webapp: save Fuzz Introspector reports in a .zip file (#1326)

This makes all introspector data available through the API, which can be useful for experimenation and exploration. Discussed in #1323 Ref: #1323 (comment) Signed-off-by: David Korczynski <[email protected]>
ossf · Nov 24, 2023 · 10c30a6 · 10c30a6
1 parent b45349a
commit 10c30a6
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 0 deletions.
diff --git a/tools/web-fuzzing-introspection/app/static/assets/db/.gitignore b/tools/web-fuzzing-introspection/app/static/assets/db/.gitignore
@@ -3,3 +3,4 @@ backup
 outdir*
 oss-fuzz-clone
 db-archive.zip
+raw-introspector-reports.zip
diff --git a/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py b/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py
@@ -32,6 +32,7 @@
 DB_JSON_ALL_CURRENT_FUNCS = 'all-project-current.json'
 DB_JSON_ALL_BRANCH_BLOCKERS = 'all-branch-blockers.json'
 DB_BUILD_STATUS_JSON = 'build-status.json'
+DB_RAW_INTROSPECTOR_REPORTS = 'raw-introspector-reports'
 
 ALL_JSON_FILES = [
     DB_JSON_DB_TIMESTAMP,
@@ -266,6 +267,16 @@ def rename_annotated_cfg(original_annotated_cfg):
     return new_annotated_cfg
 
 
+def save_fuzz_introspector_report(introspector_report, project_name, date_str):
+    if not os.path.isdir(DB_RAW_INTROSPECTOR_REPORTS):
+        os.mkdir(DB_RAW_INTROSPECTOR_REPORTS)
+
+    report_dst = os.path.join(DB_RAW_INTROSPECTOR_REPORTS,
+                              '%s-%s.json' % (project_name, date_str))
+    with open(report_dst, 'w') as report_fd:
+        json.dump(introspector_report, report_fd)
+
+
 def extract_project_data(project_name, date_str, should_include_details,
                          manager_return_dict):
     """
@@ -307,6 +318,9 @@ def extract_project_data(project_name, date_str, should_include_details,
     introspector_report_url = get_introspector_report_url_report(
         project_name, date_str.replace("-", ""))
 
+    # Save the report
+    save_fuzz_introspector_report(introspector_report, project_name, date_str)
+
     # Currently, we fail if any of code_coverage_summary of introspector_report is
     # None. This should later be adjusted such that we can continue if we only
     # have code coverage but no introspector data. However, we need to adjust
@@ -691,6 +705,14 @@ def update_db_files(db_timestamp, project_timestamps, function_list,
                              DB_JSON_ALL_PROJECT_TIMESTAMP,
                              compress_type=zipfile.ZIP_DEFLATED)
 
+        # ZIP the archived introspector reports
+        shutil.make_archive(DB_RAW_INTROSPECTOR_REPORTS, 'zip',
+                            DB_RAW_INTROSPECTOR_REPORTS)
+
+    # Cleanup DB_RAW_INTROSPECTOR_REPORTS
+    if os.path.isdir(DB_RAW_INTROSPECTOR_REPORTS):
+        shutil.rmtree(DB_RAW_INTROSPECTOR_REPORTS)
+
 
 def update_build_status(build_dict):
     with open(DB_BUILD_STATUS_JSON, "w") as f: