FuzzILLI源码-main.swift

2022-06-16
对FuzzILLI源码的main.swift的粗略阅读
main.swift

源代码的入口点main.swift位于项目的Sources/FuzzilliCli/目录下：
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import Foundation
import Fuzzilli

//
// Process commandline arguments.
//
let args = Arguments.parse(from: CommandLine.arguments)

// 检查参数是否需要打印help，Arguments类位于Sources/Fuzzilli/Util/Arguments.swift里
if args["-h"] != nil || args["--help"] != nil || args.numPositionalArguments != 1 {
	// .... //
}

let jsShellPath = args[0]

if !FileManager.default.fileExists(atPath: jsShellPath) {
    print("Invalid JS shell path \"\(jsShellPath)\", file does not exist")
    exit(-1)
}
// ... //
// 这中间都是对我们传入的命令行参数进行解析，配置fuzzer用到的各种相关的参数，如numJobs，engineName，timeout等等，如果用户没有指定的话就用默认值
// ... //
// Forbid this configuration as runtime types collection requires the AbstractInterpreter
if disableAbstractInterpreter, collectRuntimeTypes {
    print(
        """
        It is not possible to disable abstract interpretation and enable runtime types collection at the same time.
        Remove at least one of the arguments:
        --noAbstractInterpretation
        --collectRuntimeTypes
        """
    )
    exit(-1)
}

//
// Construct a fuzzer instance.
//
// 该函数创建一个fuzzer的实例，后续代码都是依赖该fuzzer实例展开的
func makeFuzzer(for profile: Profile, with configuration: Configuration) -> Fuzzer {
    // REPRL类，位于Sources\Fuzzilli\Execution\REPRL.swift，作用是用插桩过的js引擎来执行js代码
    // A script runner to execute JavaScript code in an instrumented JS engine.
    let runner = REPRL(executable: jsShellPath, processArguments: profile.processArguments, processEnvironment: profile.processEnv)
    // FuzzEngine类，位于Sources\Fuzzilli\Core\FuzzEngine.swift，同目录下还有MultiEngine.swift，HybridEngine.swift,
    // MutationEngine.swift对应以下3个case分支，默认配置下是MutationEngine
    let engine: FuzzEngine
    switch engineName {
    case "hybrid":
        engine = HybridEngine(numConsecutiveMutations: consecutiveMutations)
    case "multi":
        let mutationEngine = MutationEngine(numConsecutiveMutations: consecutiveMutations)
        let hybridEngine = HybridEngine(numConsecutiveMutations: consecutiveMutations)
        let engines = WeightedList<FuzzEngine>([
            (mutationEngine, 1),
            (hybridEngine, 1),
        ])
        engine = MultiEngine(engines: engines, initialActive: hybridEngine, iterationsPerEngine: 1000)
    default:
        engine = MutationEngine(numConsecutiveMutations: consecutiveMutations)
    }

    // Code generators to use.
    // profile位于Sources\FuzzilliCli\Profiles\目录下，比如我们要fuzz v8，那就是V8Profile.swift里的配置
    let disabledGenerators = Set(profile.disabledCodeGenerators)
    var codeGenerators = profile.additionalCodeGenerators
    // CodeGenerators位于Sources\Fuzzilli\Core\CodeGenerator.swift和 Sources\Fuzzilli\Core\CodeGenerators.swift
    // 用于生成FUZZIL的各种代码（单条指令）
    for generator in CodeGenerators {
        if disabledGenerators.contains(generator.name) {
            continue
        }
        guard let weight = codeGeneratorWeights[generator.name] else {
            print("Missing weight for code generator \(generator.name) in CodeGeneratorWeights.swift")
            exit(-1)
        }

        codeGenerators.append(generator, withWeight: weight)
    }

    // Program templates to use.
    var programTemplates = profile.additionalProgramTemplates
    // ProgramTemplates位于Sources\Fuzzilli\Core\ProgramTemplate.swift和Sources\Fuzzilli\Core\ProgramTemplates.swift
    // 用于定义生成代码的模板
    for template in ProgramTemplates {
        guard let weight = programTemplateWeights[template.name] else {
            print("Missing weight for program template \(template.name) in ProgramTemplateWeights.swift")
            exit(-1)
        }

        programTemplates.append(template, withWeight: weight)
    }
	// JavaScriptEnvironment位于Sources\Fuzzilli\Core\Environment.swift和Sources\Fuzzilli\Core\JavaScriptEnvironment.swift
    // 定义js代码能访问到的全局对象之类的，如DataView，Array这些
    // The environment containing available builtins, property names, and method names.
    let environment = JavaScriptEnvironment(additionalBuiltins: profile.additionalBuiltins, additionalObjectGroups: [])
	// JavaScriptLifter类位于Sources\Fuzzilli\Lifting\JavaScriptLifter.swift
    // lifter用于将FuzzIL programs转换成js代码
    // A lifter to translate FuzzIL programs to JavaScript.
    let lifter = JavaScriptLifter(prefix: profile.codePrefix,
                                  suffix: profile.codeSuffix,
                                  inliningPolicy: InlineOnlyLiterals(),
                                  ecmaVersion: profile.ecmaVersion)
    
	// ProgramCoverageEvaluator位于Sources\Fuzzilli\Evaluation\ProgramCoverageEvaluator.swift
    // 其内部会用到libcoverage里的函数来获取js shell程序的覆盖率相关信息，libcoverage位于Sources\libcoverage\coverage.c
    // The evaluator to score produced samples.
    let evaluator = ProgramCoverageEvaluator(runner: runner, maxResetCount: UInt64(maxResetCount))
	
    // 选择物料算法，好像是这样
    // Corpus managing interesting programs that have been found during fuzzing.
    let corpus: Corpus
    switch corpusName {
    case "basic":
        corpus = BasicCorpus(minSize: minCorpusSize, maxSize: maxCorpusSize, minMutationsPerSample: minMutationsPerSample)
    case "markov":
        corpus = MarkovCorpus(covEvaluator: evaluator as ProgramCoverageEvaluator, dropoutRate: markovDropoutRate)
    default:
        logger.fatal("Invalid corpus name provided")
    }

    // Minimizer to minimize crashes and interesting programs.
    let minimizer = Minimizer()

    // 这个是给各个Mutator进行权重的配比
    /// The mutation fuzzer responsible for mutating programs from the corpus and evaluating the outcome.
    let mutators = WeightedList([
        (CodeGenMutator(),                  3),
        (InputMutator(isTypeAware: false),  2),
        (InputMutator(isTypeAware: true),   1),
        // Can be enabled for experimental use, ConcatMutator is a limited version of CombineMutator
        // (ConcatMutator(),                1),
        (OperationMutator(),                1),
        (CombineMutator(),                  1),
        (JITStressMutator(),                1),
    ])
	
    // Fuzzer类位于Sources\Fuzzilli\Fuzzer.swift
    // Construct the fuzzer instance.
    return Fuzzer(configuration: config,
                  scriptRunner: runner,
                  engine: engine,
                  mutators: mutators,
                  codeGenerators: codeGenerators,
                  programTemplates: programTemplates,
                  evaluator: evaluator,
                  environment: environment,
                  lifter: lifter,
                  corpus: corpus,
                  deterministicCorpus: !noDeterministicCorpus,
                  minDeterminismExecs: minDeterminismExecs,
                  maxDeterminismExecs: maxDeterminismExecs,
                  minimizer: minimizer)
}

// Configuration类位于Sources\Fuzzilli\Configuration.swift
// The configuration of this fuzzer.
let config = Configuration(timeout: UInt32(timeout),
                           logLevel: logLevel,
                           crashTests: profile.crashTests,
                           isMaster: networkMasterParams != nil,
                           isWorker: networkWorkerParams != nil,
                           isFuzzing: !dontFuzz,
                           minimizationLimit: minimizationLimit,
                           useAbstractInterpretation: !disableAbstractInterpreter,
                           collectRuntimeTypes: collectRuntimeTypes,
                           enableDiagnostics: diagnostics,
                           inspection: inspectionOptions)

// 创建好fuzzer实例
let fuzzer = makeFuzzer(for: profile, with: config)

// Create a "UI". We do this now, before fuzzer initialization, so
// we are able to print log messages generated during initialization.
let ui = TerminalUI(for: fuzzer)

let logger = Logger(withLabel: "Cli")

// Remaining fuzzer initialization must happen on the fuzzer's dispatch queue.
fuzzer.sync {
    // Moudles位于Sources\Fuzzilli\Modules下，
    // 目录底下定义了NetWorksync,Statistics,Storage,Threadsync这4个moudle
    // Always want some statistics.
    fuzzer.addModule(Statistics())

    // Check core file generation on linux, prior to moving corpus file directories
    fuzzer.checkCoreFileGeneration()

    // Store samples to disk if requested.
    if let path = storagePath {
        if resume {
            // Move the old corpus to a new directory from which the files will be imported afterwards
            // before the directory is deleted.
            do {
                try FileManager.default.moveItem(atPath: path + "/corpus", toPath: path + "/old_corpus")
            } catch {
                logger.info("Nothing to resume from: \(path)/corpus does not exist")
                resume = false
            }
        } else if overwrite {
            logger.info("Deleting all files in \(path) due to --overwrite")
            try? FileManager.default.removeItem(atPath: path)
        } else {
            // The corpus directory mus be empty. We already checked this above, so just assert here
            let directory = (try? FileManager.default.contentsOfDirectory(atPath: path + "/corpus")) ?? []
            assert(directory.isEmpty)
        }

        fuzzer.addModule(Storage(for: fuzzer,
                                 storageDir: path,
                                 statisticsExportInterval: exportStatistics ? 10 * Minutes : nil
        ))
    }

    // Synchronize over the network if requested.
    if let (listenHost, listenPort) = networkMasterParams {
        fuzzer.addModule(NetworkMaster(for: fuzzer, address: listenHost, port: listenPort))
    }
    if let (masterHost, masterPort) = networkWorkerParams {
        fuzzer.addModule(NetworkWorker(for: fuzzer, hostname: masterHost, port: masterPort))
    }

    // Synchronize with thread workers if requested.
    if numJobs > 1 {
        fuzzer.addModule(ThreadMaster(for: fuzzer))
    }

    // Check for potential misconfiguration.
    if !config.isWorker && storagePath == nil {
        logger.warning("No filesystem storage configured, found crashes will be discarded!")
    }

    // Exit this process when the main fuzzer stops.
    fuzzer.registerEventListener(for: fuzzer.events.ShutdownComplete) { reason in
        exit(reason.toExitCode())
    }

    // Initialize the fuzzer, and run startup tests
    // initialize函数里会调用fuzzer实例里各个对象的initialize函数：
    // public func initialize() {
    // Initialize the script runner first so we are able to execute programs.
    //    runner.initialize(with: self)

    //    // Then initialize all components.
    //    engine.initialize(with: self)
    //    evaluator.initialize(with: self)
    //    environment.initialize(with: self)
    //    corpus.initialize(with: self)
    //    minimizer.initialize(with: self)

    //    // Finally initialize all modules.
    //    for module in modules.values {
    //        module.initialize(with: self)
    //    }
   	//    // ... //
    //   } // end Fuzzer.initialize function
    fuzzer.initialize()
    fuzzer.runStartupTests()
}

// Add thread worker instances if requested
//
// This happens here, before any corpus is imported, so that any imported programs are
// forwarded to the ThreadWorkers automatically when they are deemed interesting.
//
// This must *not* happen on the main fuzzer's queue since workers perform synchronous
// operations on the master's dispatch queue.
// 如果numJobs大于1，就创建numJobs个fuzzer实例
var instances = [fuzzer]
for _ in 1..<numJobs {
    let worker = makeFuzzer(for: profile, with: config)
    instances.append(worker)
    let g = DispatchGroup()

    g.enter()
    worker.sync {
        worker.addModule(Statistics())
        worker.addModule(ThreadWorker(forMaster: fuzzer))
        worker.registerEventListener(for: worker.events.Initialized) { g.leave() }
        worker.initialize()
    }

    // Wait for the worker to be fully initialized
    g.wait()
}

// 导入物料
// Import a corpus if requested and start the main fuzzer instance.
fuzzer.sync {
    func loadCorpus(from dirPath: String) -> [Program] {
        var isDir: ObjCBool = false
        if !FileManager.default.fileExists(atPath: dirPath, isDirectory:&isDir) || !isDir.boolValue {
            logger.fatal("Cannot import programs from \(dirPath), it is not a directory!")
        }

        var programs = [Program]()
        let fileEnumerator = FileManager.default.enumerator(atPath: dirPath)
        while let filename = fileEnumerator?.nextObject() as? String {
            guard filename.hasSuffix(".fuzzil.protobuf") else { continue }
            let path = dirPath + "/" + filename
            do {
                let data = try Data(contentsOf: URL(fileURLWithPath: path))
                let pb = try Fuzzilli_Protobuf_Program(serializedData: data)
                let program = try Program.init(from: pb)
                programs.append(program)
            } catch {
                logger.error("Failed to load program \(path): \(error). Skipping")
            }
        }

        return programs
    }

    // Resume a previous fuzzing session if requested
    if resume, let path = storagePath {
        logger.info("Resuming previous fuzzing session. Importing programs from corpus directory now. This may take some time")
        let corpus = loadCorpus(from: path + "/old_corpus")

        // Delete the old corpus directory now
        try? FileManager.default.removeItem(atPath: path + "/old_corpus")

        fuzzer.importCorpus(corpus, importMode: .interestingOnly(shouldMinimize: false))  // We assume that the programs are already minimized
        logger.info("Successfully resumed previous state. Corpus now contains \(fuzzer.corpus.size) elements")
    }

    // Import a full corpus if requested
    if let path = corpusImportAllPath {
        let corpus = loadCorpus(from: path)
        logger.info("Starting All-corpus import of \(corpus.count) programs. This may take some time")
        fuzzer.importCorpus(corpus, importMode: .all)
        logger.info("Successfully imported \(path). Corpus now contains \(fuzzer.corpus.size) elements")
    }

    // Import a coverage-only corpus if requested
    if let path = corpusImportCovOnlyPath {
        var corpus = loadCorpus(from: path)
        // Sorting the corpus helps avoid minimizing large programs that produce new coverage due to small snippets also included by other, smaller samples
        corpus.sort(by: { $0.size < $1.size })
        logger.info("Starting Cov-only corpus import of \(corpus.count) programs. This may take some time")
        fuzzer.importCorpus(corpus, importMode: .interestingOnly(shouldMinimize: true))
        logger.info("Successfully imported \(path). Samples will be added to the corpus once they are minimized")
    }
    
    // Import and merge an existing corpus if requested
    if let path = corpusImportMergePath {
        let corpus = loadCorpus(from: path)
        logger.info("Starting corpus merge of \(corpus.count) programs. This may take some time")
        fuzzer.importCorpus(corpus, importMode: .interestingOnly(shouldMinimize: false))
        logger.info("Successfully imported \(path). Corpus now contains \(fuzzer.corpus.size) elements")
    }
}

// Install signal handlers to terminate the fuzzer gracefully.
var signalSources: [DispatchSourceSignal] = []
for sig in [SIGINT, SIGTERM] {
    // Seems like we need this so the dispatch sources work correctly?
    signal(sig, SIG_IGN)

    let source = DispatchSource.makeSignalSource(signal: sig, queue: DispatchQueue.main)
    source.setEventHandler {
        fuzzer.async {
            fuzzer.shutdown(reason: .userInitiated)
        }
    }
    source.activate()
    signalSources.append(source)
}

#if !os(Windows)
// Install signal handler for SIGUSR1 to print the next program that is generated.
signal(SIGUSR1, SIG_IGN)
let source = DispatchSource.makeSignalSource(signal: SIGUSR1, queue: DispatchQueue.main)
source.setEventHandler {
    ui.printNextGeneratedProgram = true
}
source.activate()
signalSources.append(source)
#endif

// Finally, start fuzzing.
for fuzzer in instances {
    fuzzer.sync {
        // 调用start函数，开始真正的fuzz
        fuzzer.start(runFor: numIterations)
    }
}

// Start dispatching tasks on the main queue.
RunLoop.main.run()
main.swift大体就是这样，但是其实Fuzzer类里有很多组件，fuzz就是靠这些组件来相互配合进行的，所以后续就是要深入这些组件的源代码。看其是如何运作的
参考链接

https://github.com/googleprojectzero/fuzzilli/blob/main/Docs/HowFuzzilliWorks.md