Real-time collaboration for Jupyter Notebooks, Linux Terminals, LaTeX, VS Code, R IDE, and more,
all in one place.
Real-time collaboration for Jupyter Notebooks, Linux Terminals, LaTeX, VS Code, R IDE, and more,
all in one place.
Path: blob/master/docs/build.rb
Views: 11704
require 'fileutils'1require 'uri'2require 'open3'3require 'optparse'4require 'did_you_mean'5require 'kramdown'6require_relative './navigation'78# This build module was used to migrate the old Metasploit wiki https://github.com/rapid7/metasploit-framework/wiki into a format9# supported by Jekyll. Jekyll was chosen as it was written in Ruby, which should reduce the barrier to entry for contributions.10#11# The build script took the flatlist of markdown files from the wiki, and converted them into the hierarchical folder structure12# for nested documentation. This configuration is defined in `navigation.rb`13#14# In the future a different site generator could be used, but it should be possible to use this build script again to migrate to a new format15#16# For now the doc folder only contains the key files for building the docs site and no content. The content is created on demand17# from the `metasploit-framework.wiki` folder on each build18module Build19# The metasploit-framework.wiki files that are committed to Metasploit framework's repository20WIKI_PATH = 'metasploit-framework.wiki'.freeze21# A locally cloned version of https://github.com/rapid7/metasploit-framework/wiki - should no longer be required for normal workflows22OLD_WIKI_PATH = 'metasploit-framework.wiki.old'.freeze23RELEASE_BUILD_ARTIFACTS = '_site'.freeze2425# For now we Git clone the existing metasploit wiki and generate the Jekyll markdown files26# for each build. This allows changes to be made to the existing wiki until it's migrated27# into the main framework repo28module Git29def self.clone_wiki!30unless File.exist?(OLD_WIKI_PATH)31Build.run_command "git clone https://github.com/rapid7/metasploit-framework.wiki.git #{OLD_WIKI_PATH}", exception: true32end3334Build.run_command "cd #{OLD_WIKI_PATH}; git pull", exception: true35end36end3738class ConfigValidationError < StandardError39end4041# Configuration for generating the new website hierarchy, from the existing metasploit-framework wiki42class Config43include Enumerable4445def initialize(config)46@config = config47end4849def validate!50configured_paths = all_file_paths51missing_paths = available_paths.map { |path| path.gsub("#{WIKI_PATH}/", '') } - ignored_paths - existing_docs - configured_paths52raise ConfigValidationError, "Unhandled paths #{missing_paths.join(', ')} - add navigation entries to navigation.rb for these files" if missing_paths.any?5354each do |page|55page_keys = page.keys56allowed_keys = %i[old_wiki_path path new_base_name nav_order title new_path folder children has_children parents]57invalid_keys = page_keys - allowed_keys5859suggestion = DidYouMean::SpellChecker.new(dictionary: allowed_keys).correct(invalid_keys[0]).first60error = "#{page} had invalid keys #{invalid_keys.join(', ')}."61error += " Did you mean #{suggestion}?" if suggestion6263raise ConfigValidationError, error if invalid_keys.any?64end6566# Ensure unique folder names67folder_titles = to_enum.select { |page| page[:folder] }.map { |page| page[:title] }68duplicate_folder = folder_titles.tally.select { |_name, count| count > 1 }69raise ConfigValidationError, "Duplicate folder titles, will cause issues: #{duplicate_folder}" if duplicate_folder.any?7071# Ensure no folder titles match file titles72page_titles = to_enum.reject { |page| page[:folder] }.map { |page| page[:title] }73title_collisions = (folder_titles & page_titles).tally74raise ConfigValidationError, "Duplicate folder/page titles, will cause issues: #{title_collisions}" if title_collisions.any?7576# Ensure there are no files being migrated to multiple places77page_paths = to_enum.reject { |page| page[:path] }.map { |page| page[:title] }78duplicate_page_paths = page_paths.tally.select { |_name, count| count > 1 }79raise ConfigValidationError, "Duplicate paths, will cause issues: #{duplicate_page_paths}" if duplicate_page_paths.any?8081# Ensure new file paths are only alphanumeric and hyphenated82new_paths = to_enum.map { |page| page[:new_path] }83invalid_new_paths = new_paths.reject { |path| File.basename(path) =~ /^[a-zA-Z0-9_-]*\.md$/ }84raise ConfigValidationError, "Only alphanumeric and hyphenated file names required: #{invalid_new_paths}" if invalid_new_paths.any?85end8687def available_paths88Dir.glob("#{WIKI_PATH}/**/*{.md,.textile}", File::FNM_DOTMATCH)89end9091def ignored_paths92[93]94end9596def existing_docs97existing_docs = Dir.glob('docs/**/*', File::FNM_DOTMATCH)98existing_docs99end100101def each(&block)102config.each do |parent|103recurse(with_metadata(parent), &block)104end105end106107def all_file_paths108to_enum.map { |item| item[:path] }.to_a109end110111protected112113# depth first traversal114def recurse(parent_with_metadata, &block)115block.call(parent_with_metadata)116parent_with_metadata[:children].to_a.each do |child|117child_with_metadata = with_metadata(child, parents: parent_with_metadata[:parents] + [parent_with_metadata])118recurse(child_with_metadata, &block)119end120end121122def with_metadata(child, parents: [])123child = child.clone124125if child[:folder]126parent_folders = parents.map { |page| page[:folder] }127child[:new_path] = File.join(*parent_folders, child[:folder], 'index.md')128else129path = child[:path]130base_name = child[:new_base_name] || File.basename(path)131132# title calculation133computed_title = File.basename(base_name, '.md').gsub('-', ' ')134if child[:title].is_a?(Proc)135child[:title] = child[:title].call(computed_title)136else137child[:title] ||= computed_title138end139140parent_folders = parents.map { |page| page[:folder] }141child[:new_path] = File.join(*parent_folders, base_name.downcase)142end143144child[:parents] = parents145child[:has_children] = true if child[:children].to_a.any?146147child148end149150attr_reader :config151end152153# Extracts markdown links from https://github.com/rapid7/metasploit-framework/wiki into a Jekyll format154# Additionally corrects links to Github155class LinkCorrector156def initialize(config)157@config = config158@links = {}159end160161def syntax_errors_for(markdown)162MarkdownLinkSyntaxVerifier.errors_for(markdown)163end164165def extract(markdown)166extracted_absolute_wiki_links = extract_absolute_wiki_links(markdown)167@links = @links.merge(extracted_absolute_wiki_links)168169extracted_relative_links = extract_relative_links(markdown)170@links = @links.merge(extracted_relative_links)171172@links173end174175def rerender(markdown)176links ||= @links177178new_markdown = markdown.clone179links.each_value do |link|180new_markdown.gsub!(link[:full_match], link[:replacement])181end182183new_markdown184end185186attr_reader :links187188protected189190def pages191@config.enum_for(:each).map { |page| page }192end193194# scans for absolute links to the old wiki such as 'https://docs.metasploit.com/docs/using-metasploit/advanced/metasploit-web-service.html'195def extract_absolute_wiki_links(markdown)196new_links = {}197198markdown.scan(%r{(https?://github.com/rapid7/metasploit-framework/wiki/([\w().%_#-]+))}) do |full_match, old_path|199full_match = full_match.gsub(/[).]+$/, '')200old_path = URI.decode_www_form_component(old_path.gsub(/[).]+$/, ''))201202begin203old_path_anchor = URI.parse(old_path).fragment204rescue URI::InvalidURIError205old_path_anchor = nil206end207208new_path = new_path_for(old_path, old_path_anchor)209replacement = "{% link docs/#{new_path} %}#{old_path_anchor ? "##{old_path_anchor}" : ""}"210211link = {212full_match: full_match,213type: :absolute,214new_path: new_path,215replacement: replacement216}217218new_links[full_match] = link219end220221new_links222end223224# Scans for Github wiki flavor links such as:225# '[[Relative Path]]'226# '[[Custom name|Relative Path]]'227# '[[Custom name|relative-path]]'228# '[[Custom name|./relative-path.md]]'229# '[[Custom name|./relative-path.md#section-anchor-to-link-to]]'230# Note that the page target resource file is validated for existence at build time - but the section anchors are not231def extract_relative_links(markdown)232existing_links = @links233new_links = {}234235markdown.scan(/(\[\[([\w\/_ '().:,-]+)(?:\|([\w\/_ '():,.#-]+))?\]\])/) do |full_match, left, right|236old_path = (right || left)237begin238old_path_anchor = URI.parse(old_path).fragment239rescue URI::InvalidURIError240old_path_anchor = nil241end242new_path = new_path_for(old_path, old_path_anchor)243if existing_links[full_match] && existing_links[full_match][:new_path] != new_path244raise "Link for #{full_match} previously resolved to #{existing_links[full_match][:new_path]}, but now resolves to #{new_path}"245end246247link_text = left248replacement = "[#{link_text}]({% link docs/#{new_path} %}#{old_path_anchor ? "##{old_path_anchor}" : ""})"249250link = {251full_match: full_match,252type: :relative,253left: left,254right: right,255new_path: new_path,256replacement: replacement257}258259new_links[full_match] = link260end261262new_links263end264265def new_path_for(old_path, old_path_anchor)266# Strip out any leading `./` or `/` before the relative path.267# This is needed for our later code that does additional filtering for268# potential ambiguity with absolute paths since those comparisons occur269# against filenames without the leading ./ and / parts.270old_path = old_path.gsub(/^[.\/]+/, '')271272# Replace any spaces in the file name with - separators, then273# make replace anchors with an empty string.274old_path = old_path.gsub(' ', '-').gsub("##{old_path_anchor}", '')275276matched_pages = pages.select do |page|277!page[:folder] &&278(File.basename(page[:path]).downcase == "#{File.basename(old_path)}.md".downcase ||279File.basename(page[:path]).downcase == "#{File.basename(old_path)}".downcase)280end281if matched_pages.empty?282raise "Link not found: #{old_path}"283end284# Additional filter for absolute paths if there's potential ambiguity285if matched_pages.count > 1286refined_pages = matched_pages.select do |page|287!page[:folder] &&288(page[:path].downcase == "#{old_path}.md".downcase ||289page[:path].downcase == old_path.downcase)290end291292if refined_pages.count != 1293page_paths = matched_pages.map { |page| page[:path] }294raise "Duplicate paths for #{old_path} - possible page paths found: #{page_paths}"295end296297matched_pages = refined_pages298end299300matched_pages.first.fetch(:new_path)301end302end303304# Verifies that markdown links are not relative. Instead the Github wiki flavored syntax should be used.305#306# Example bad: `[Human readable text](./some-documentation-link)`307# Example good: `[[Human readable text|./some-documentation-link]]`308class MarkdownLinkSyntaxVerifier309# Detects the usage of bad syntax and returns an array of detected errors310#311# @param [String] markdown The markdown312# @return [Array<String>] An array of human readable errors that should be resolved313def self.errors_for(markdown)314document = Kramdown::Document.new(markdown)315document.to_validated_wiki_page316warnings = document.warnings.select { |warning| warning.start_with?(Kramdown::Converter::ValidatedWikiPage::WARNING_PREFIX) }317warnings318end319320# Implementation detail: There doesn't seem to be a generic AST visitor pattern library for Ruby; We instead implement321# Kramdown's Markdown to HTML Converter API, override the link converter method, and warn on any invalid links that are identified.322# The {MarkdownLinkVerifier} will ignore the HTML result, and return any detected errors instead.323#324# https://kramdown.gettalong.org/rdoc/Kramdown/Converter/Html.html325class Kramdown::Converter::ValidatedWikiPage < Kramdown::Converter::Html326WARNING_PREFIX = '[WikiLinkValidation]'327328def convert_a(el, indent)329link_href = el.attr['href']330if relative_link?(link_href)331link_text = el.children.map { |child| convert(child) }.join332warning "Invalid docs link syntax found on line #{el.options[:location]}: Invalid relative link #{link_href} found. Please use the syntax [[#{link_text}|#{link_href}]] instead"333end334335if absolute_docs_link?(link_href)336begin337example_path = ".#{URI.parse(link_href).path}"338rescue URI::InvalidURIError339example_path = "./path-to-markdown-file"340end341342link_text = el.children.map { |child| convert(child) }.join343warning "Invalid docs link syntax found on line #{el.options[:location]}: Invalid absolute link #{link_href} found. Please use relative links instead, i.e. [[#{link_text}|#{example_path}]] instead"344end345346super347end348349private350351def warning(text)352super "#{WARNING_PREFIX} #{text}"353end354355def relative_link?(link_path)356!(link_path.start_with?('http:') || link_path.start_with?('https:') || link_path.start_with?('mailto:') || link_path.start_with?('#'))357end358359# @return [TrueClass, FalseClass] True if the link is to a Metasploit docs page that isn't either the root home page or the API site, otherwise false360def absolute_docs_link?(link_path)361link_path.include?('docs.metasploit.com') && !link_path.include?('docs.metasploit.com/api') && !(link_path == 'https://docs.metasploit.com/')362end363end364end365366# Parses a wiki page and can add/remove/update a deprecation notice367class WikiDeprecationText368MAINTAINER_MESSAGE_PREFIX = "<!-- Maintainers: "369private_constant :MAINTAINER_MESSAGE_PREFIX370371USER_MESSAGE_PREFIX = '**Documentation Update:'.freeze372private_constant :USER_MESSAGE_PREFIX373374def self.upsert(original_wiki_content, old_path:, new_url:)375history_link = old_path.include?("#{WIKI_PATH}/Home.md") ? './Home/_history' : './_history'376maintainer_message = "#{MAINTAINER_MESSAGE_PREFIX} Please do not modify this file directly, create a pull request instead -->\n\n"377user_message = "#{USER_MESSAGE_PREFIX} This Wiki page should be viewable at [#{new_url}](#{new_url}). Or if it is no longer available, see this page's [previous history](#{history_link})**\n\n"378deprecation_text = maintainer_message + user_message379"#{deprecation_text}"380end381382def self.remove(original_wiki_content)383original_wiki_content384.gsub(/^#{Regexp.escape(MAINTAINER_MESSAGE_PREFIX)}.*$\s+/, '')385.gsub(/^#{Regexp.escape(USER_MESSAGE_PREFIX)}.*$\s+/, '')386end387end388389# Converts Wiki markdown pages into a valid Jekyll format390class WikiMigration391# Implements two core components:392# - Converts the existing Wiki markdown pages into a Jekyll format393# - Optionally updates the existing Wiki markdown pages with a link to the new website location394def run(config, options = {})395begin396config.validate!397rescue398puts "[!] Validation failed. Please verify navigation.rb is valid, as well as the markdown file"399raise400end401402# Clean up new docs folder in preparation for regenerating it entirely from the latest wiki403result_folder = File.join('.', 'docs')404FileUtils.remove_dir(result_folder, true)405FileUtils.mkdir(result_folder)406407link_corrector = link_corrector_for(config)408config.each do |page|409page_config = {410layout: 'default',411**page.slice(:title, :has_children, :nav_order),412parent: (page[:parents][-1] || {})[:title],413warning: "Do not modify this file directly. Please modify metasploit-framework/docs/metasploit-framework.wiki instead",414old_path: page[:path] ? File.join(WIKI_PATH, page[:path]) : "none - folder automatically generated",415has_content: !page[:path].nil?416}.compact417418page_config[:has_children] = true if page[:has_children]419preamble = <<~PREAMBLE420---421#{page_config.map { |key, value| "#{key}: #{value.to_s.strip.inspect}" }.join("\n")}422---423424PREAMBLE425426new_path = File.join(result_folder, page[:new_path])427FileUtils.mkdir_p(File.dirname(new_path))428429if page[:folder] && page[:path].nil?430new_docs_content = preamble.rstrip + "\n"431else432old_path = File.join(WIKI_PATH, page[:path])433previous_content = File.read(old_path, encoding: Encoding::UTF_8)434new_docs_content = preamble + WikiDeprecationText.remove(previous_content)435new_docs_content = link_corrector.rerender(new_docs_content)436437# Update the old Wiki with links to the new website438if options[:update_wiki_deprecation_notice]439new_url = options[:update_wiki_deprecation_notice][:new_website_url]440if page[:new_path] != 'home.md'441new_url += 'docs/' + page[:new_path].gsub('.md', '.html')442end443updated_wiki_content = WikiDeprecationText.upsert(previous_content, old_path: old_path, new_url: new_url)444old_wiki_path = File.join(WIKI_PATH, page[:path])445File.write(old_wiki_path, updated_wiki_content, mode: 'w', encoding: Encoding::UTF_8)446end447end448449File.write(new_path, new_docs_content, mode: 'w', encoding: Encoding::UTF_8)450end451452# Now that the docs folder is created, time to move the home.md file out453FileUtils.mv('docs/home.md', 'index.md')454end455456protected457458def link_corrector_for(config)459link_corrector = LinkCorrector.new(config)460errors = []461config.each do |page|462unless page[:path].nil?463content = File.read(File.join(WIKI_PATH, page[:path]), encoding: Encoding::UTF_8)464syntax_errors = link_corrector.syntax_errors_for(content)465errors << { path: page[:path], messages: syntax_errors } if syntax_errors.any?466467link_corrector.extract(content)468end469end470471if errors.any?472errors.each do |error|473$stderr.puts "[!] Error #{File.join(WIKI_PATH, error[:path])}:\n#{error[:messages].map { |message| "\t- #{message}\n" }.join}"474end475476raise "Errors found in markdown syntax"477end478479link_corrector480end481end482483# Serve the release build at http://127.0.0.1:4000/metasploit-framework/484class ReleaseBuildServer485autoload :WEBrick, 'webrick'486487def self.run488server = WEBrick::HTTPServer.new(489{490Port: 4000491}492)493server.mount('/', WEBrick::HTTPServlet::FileHandler, RELEASE_BUILD_ARTIFACTS)494trap('INT') do495server.shutdown496rescue StandardError497nil498end499server.start500ensure501server.shutdown502end503end504505def self.run_command(command, exception: true)506puts "[*] #{command}"507result = ''508::Open3.popen2e(509{ 'BUNDLE_GEMFILE' => File.join(Dir.pwd, 'Gemfile') },510'/bin/bash', '--login', '-c', command511) do |stdin, stdout_and_stderr, wait_thread|512stdin.close_write513514while wait_thread.alive?515ready = IO.select([stdout_and_stderr], nil, nil, 1)516517next unless ready518reads, _writes, _errors = ready519520reads.to_a.each do |io|521data = io.read_nonblock(1024)522puts data523result += data524rescue EOFError, Errno::EAGAIN525# noop526end527end528529if !wait_thread.value.success? && exception530raise "command #{command.inspect} did not succeed, exit status #{wait_thread.value.exitstatus.inspect}"531end532end533534result535end536537def self.run(options)538Git.clone_wiki! if options[:wiki_pull]539540# Create a new branch based on the commits from https://github.com/rapid7/metasploit-framework/wiki to move541# Wiki files into the metasploit-framework repo542if options[:create_wiki_to_framework_migration_branch]543starting_branch = run_command("git rev-parse --abbrev-ref HEAD").chomp544new_wiki_branch_name = "move-all-docs-into-folder"545new_framework_branch_name = "merge-metasploit-framework-wiki-into-metasploit-framework"546547begin548# Create a new folder and branch in the old metasploit wiki for where we'd like it to be inside of the metasploit-framework repo549Dir.chdir(OLD_WIKI_PATH) do550# Reset the repo back551run_command("git checkout master", exception: false)552run_command("git reset HEAD --hard", exception: false)553run_command("rm -rf metasploit-framework.wiki", exception: false)554555# Create a new folder to move the wiki contents into556FileUtils.mkdir_p("metasploit-framework.wiki")557run_command("mv *[^metasploit-framework.wiki]* metasploit-framework.wiki", exception: false)558559# Create a new branch + commit560run_command("git branch -D #{new_wiki_branch_name}", exception: false)561run_command("git checkout -b #{new_wiki_branch_name}")562run_command("git add metasploit-framework.wiki")563run_command("git commit -am 'Put markdown files into new folder metasploit-framework.wiki in preparation for migration'")564end565566# Create a new branch that can be used to create a pull request567run_command("git branch -D #{new_framework_branch_name}", exception: false)568run_command("git checkout -b #{new_framework_branch_name}")569run_command("git remote remove wiki", exception: false)570run_command("git remote add -f wiki #{File.join(Dir.pwd, OLD_WIKI_PATH)}", exception: false)571# run_command("git remote update wiki")572run_command("git merge -m 'Migrate docs from https://github.com/rapid7/metasploit-framework/wiki to main repository' wiki/#{new_wiki_branch_name} --allow-unrelated-histories")573574puts "new branch #{new_framework_branch_name} successfully created"575ensure576run_command("git checkout #{starting_branch}")577end578end579580if options[:copy_old_wiki]581FileUtils.copy_entry(OLD_WIKI_PATH, WIKI_PATH, preserve = false, dereference_root = false, remove_destination = true)582# Remove any deprecation text that might be present after copying the old wiki583Dir.glob(File.join(WIKI_PATH, '**', '*.md')) do |path|584previous_content = File.read(path, encoding: Encoding::UTF_8)585new_content = WikiDeprecationText.remove(previous_content)586587File.write(path, new_content, mode: 'w', encoding: Encoding::UTF_8)588end589end590591unless options[:build_content]592config = Config.new(NAVIGATION_CONFIG)593migrator = WikiMigration.new594migrator.run(config, options)595end596597if options[:production]598FileUtils.remove_dir(RELEASE_BUILD_ARTIFACTS, true)599run_command('JEKYLL_ENV=production bundle exec jekyll build')600601if options[:serve]602ReleaseBuildServer.run603end604elsif options[:staging]605FileUtils.remove_dir(RELEASE_BUILD_ARTIFACTS, true)606run_command('JEKYLL_ENV=production bundle exec jekyll build --config _config.yml,_config_staging.yml')607608if options[:serve]609ReleaseBuildServer.run610end611elsif options[:serve]612run_command('bundle exec jekyll serve --config _config.yml,_config_development.yml --incremental')613end614end615end616617if $PROGRAM_NAME == __FILE__618options = {619copy_old_wiki: false,620wiki_pull: false621}622options_parser = OptionParser.new do |opts|623opts.banner = "Usage: #{File.basename(__FILE__)} [options]"624625opts.on '-h', '--help', 'Help banner.' do626return print(opts.help)627end628629opts.on('--production', 'Run a production build') do |production|630options[:production] = production631end632633opts.on('--staging', 'Run a staging build for deploying to gh-pages') do |staging|634options[:staging] = staging635end636637opts.on('--serve', 'serve the docs site') do |serve|638options[:serve] = serve639end640641opts.on('--[no]-copy-old-wiki [FLAG]', TrueClass, 'Copy the content from the old wiki to the new local wiki folder') do |copy_old_wiki|642options[:copy_old_wiki] = copy_old_wiki643end644645opts.on('--[no-]-wiki-pull', FalseClass, 'Pull the Metasploit Wiki') do |wiki_pull|646options[:wiki_pull] = wiki_pull647end648649opts.on('--update-wiki-deprecation-notice [WEBSITE_URL]', 'Updates the old wiki deprecation notes') do |new_website_url|650new_website_url ||= 'https://docs.metasploit.com/'651options[:update_wiki_deprecation_notice] = {652new_website_url: new_website_url653}654end655656opts.on('--create-wiki-to-framework-migration-branch') do657options[:create_wiki_to_framework_migration_branch] = true658end659end660if ARGV.length == 0661puts options_parser.help662exit 1663end664options_parser.parse!665666Build.run(options)667end668669670