From b706ea5eff4c070cd421447f23dcfc0f3beeda97 Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Mon, 11 Nov 2019 15:26:40 +1100 Subject: [PATCH] Fix deadlock with local server ndb. (#1184) Fixes #1183. --- src/python/datastore/data_types.py | 377 ++++++++++++++++------------- 1 file changed, 202 insertions(+), 175 deletions(-) diff --git a/src/python/datastore/data_types.py b/src/python/datastore/data_types.py index 5c146054f23..90f9fbf2a6f 100644 --- a/src/python/datastore/data_types.py +++ b/src/python/datastore/data_types.py @@ -20,6 +20,8 @@ import re import six +from future import utils as future_utils + from base import json_utils from base import utils from datastore import ndb @@ -140,6 +142,31 @@ def clone_entity(e, **extra_args): return ent_class(**props) +# TODO(ochang): Remove these hacks once migrated to Python 3. +# NDB makes comparisons with the native str, unicode types, which cause issues +# when they're actually futurize's newstr. Add a validator to String and +# Text properties to coerce them to native types. +def _to_native_str(_, text): + """Convert from future's newstr to a native str.""" + return future_utils.text_to_native_str(text, encoding='utf-8') + + +class TextProperty(ndb.TextProperty): + """Overridden text property.""" + + def __init__(self, *args, **kwargs): + super(TextProperty, self).__init__( + *args, validator=_to_native_str, **kwargs) + + +class StringProperty(ndb.StringProperty): + """Overridden string property.""" + + def __init__(self, *args, **kwargs): + super(StringProperty, self).__init__( + *args, validator=_to_native_str, **kwargs) + + class SecuritySeverity(object): """Enum for Security Severity.""" CRITICAL = 0 @@ -246,10 +273,10 @@ class Model(ndb.Model): class Blacklist(Model): """Represents global blacklist to track entries for suppressions files.""" # Function name. - function_name = ndb.StringProperty() + function_name = StringProperty() # Tool name. - tool_name = ndb.StringProperty() + tool_name = StringProperty() # Testcase ID. testcase_id = ndb.IntegerProperty() @@ -264,52 +291,52 @@ class Fuzzer(Model): timestamp = ndb.DateTimeProperty() # Fuzzer Name. - name = ndb.StringProperty() + name = StringProperty() # The name of the archive that the user uploaded. - filename = ndb.StringProperty() + filename = StringProperty() # Blobstore key for this fuzzer. - blobstore_key = ndb.StringProperty() + blobstore_key = StringProperty() # String representation of the file size. - file_size = ndb.StringProperty() + file_size = StringProperty() # Fuzzer's main executable path, relative to root. - executable_path = ndb.StringProperty() + executable_path = StringProperty() # Revision number of the fuzzer. revision = ndb.IntegerProperty() # Fuzzer's source (for accountability). - source = ndb.StringProperty() + source = StringProperty() # Testcase timeout. timeout = ndb.IntegerProperty() # Supported platforms. - supported_platforms = ndb.StringProperty() + supported_platforms = StringProperty() # Custom script that should be used to launch chrome for this fuzzer. - launcher_script = ndb.StringProperty() + launcher_script = StringProperty() # Result from the last fuzzer run showing the number of testcases generated. - result = ndb.StringProperty() + result = StringProperty() # Last result update timestamp. result_timestamp = ndb.DateTimeProperty() # Console output from last fuzzer run. - console_output = ndb.TextProperty() + console_output = TextProperty() # Return code from last fuzzer run. return_code = ndb.IntegerProperty() # Blobstore key for the sample testcase generated by the fuzzer. - sample_testcase = ndb.StringProperty() + sample_testcase = StringProperty() # Job types for this fuzzer. - jobs = ndb.StringProperty(repeated=True) + jobs = StringProperty(repeated=True) # Is the fuzzer coming from an external contributor ? Useful for adding # reward flags. @@ -322,16 +349,16 @@ class Fuzzer(Model): untrusted_content = ndb.BooleanProperty(default=False) # Data bundle name. - data_bundle_name = ndb.StringProperty(default='') + data_bundle_name = StringProperty(default='') # Additional environment variables that need to be set for this fuzzer. - additional_environment_string = ndb.TextProperty() + additional_environment_string = TextProperty() # Column specification for stats. - stats_columns = ndb.StringProperty(indexed=False) + stats_columns = StringProperty(indexed=False) # Helpful descriptions for the stats_columns. In a yaml format. - stats_column_descriptions = ndb.TextProperty(indexed=False) + stats_column_descriptions = TextProperty(indexed=False) # Whether this is a builtin fuzzer. builtin = ndb.BooleanProperty(indexed=False, default=False) @@ -349,34 +376,34 @@ class BuildCrashStatsJobHistory(Model): class Testcase(Model): """Represents a single testcase.""" # Crash on an invalid read/write. - crash_type = ndb.StringProperty() + crash_type = StringProperty() # Crashing address. - crash_address = ndb.StringProperty(indexed=False) + crash_address = StringProperty(indexed=False) # First x stack frames. - crash_state = ndb.StringProperty() + crash_state = StringProperty() # Complete stacktrace. - crash_stacktrace = ndb.TextProperty(indexed=False) + crash_stacktrace = TextProperty(indexed=False) # Last tested crash stacktrace using the latest revision. - last_tested_crash_stacktrace = ndb.TextProperty(indexed=False) + last_tested_crash_stacktrace = TextProperty(indexed=False) # Blobstore keys for various things like original testcase, minimized # testcase, etc. - fuzzed_keys = ndb.StringProperty(indexed=False) - minimized_keys = ndb.StringProperty(indexed=False) - minidump_keys = ndb.StringProperty(indexed=False) + fuzzed_keys = StringProperty(indexed=False) + minimized_keys = StringProperty(indexed=False) + minidump_keys = StringProperty(indexed=False) # Tracking issue tracker bug. One bug number per line (future extension). - bug_information = ndb.StringProperty() + bug_information = StringProperty() # Regression range. - regression = ndb.StringProperty(default='') + regression = StringProperty(default='') # Revisions where this issue has been fixed. - fixed = ndb.StringProperty(default='') + fixed = StringProperty(default='') # Is it a security bug ? security_flag = ndb.BooleanProperty(default=False) @@ -388,32 +415,32 @@ class Testcase(Model): one_time_crasher_flag = ndb.BooleanProperty(default=False) # Any additional comments. - comments = ndb.TextProperty(default='', indexed=False) + comments = TextProperty(default='', indexed=False) # Revision that we discovered the crash in. crash_revision = ndb.IntegerProperty() # The file on the bot that generated the testcase. - original_absolute_path = ndb.StringProperty(indexed=False, default='') - absolute_path = ndb.StringProperty(indexed=False) + original_absolute_path = StringProperty(indexed=False, default='') + absolute_path = StringProperty(indexed=False) # Minimized argument list. - minimized_arguments = ndb.TextProperty(default='', indexed=False) + minimized_arguments = TextProperty(default='', indexed=False) # Window argument (usually width, height, top, left, etc). - window_argument = ndb.TextProperty(default='', indexed=False) + window_argument = TextProperty(default='', indexed=False) # Type of job associated with this testcase. - job_type = ndb.StringProperty() + job_type = StringProperty() # Original job queue used for tasks created for this testcase. - queue = ndb.StringProperty(indexed=False) + queue = StringProperty(indexed=False) # State representing whether the fuzzed or minimized testcases are archived. archive_state = ndb.IntegerProperty(default=0, indexed=False) # File name of the original uploaded archive. - archive_filename = ndb.StringProperty(indexed=False) + archive_filename = StringProperty(indexed=False) # Is this a binary file? binary_flag = ndb.BooleanProperty(default=False, indexed=False) @@ -428,10 +455,10 @@ class Testcase(Model): http_flag = ndb.BooleanProperty(default=False, indexed=False) # Name of the fuzzer used to generate this testcase. - fuzzer_name = ndb.StringProperty() + fuzzer_name = StringProperty() # Status of this testcase (pending, processed, unreproducible, etc). - status = ndb.StringProperty(default='Processed') + status = StringProperty(default='Processed') # Id of the testcase that this is marked as a duplicate of. duplicate_of = ndb.IntegerProperty(indexed=False) @@ -446,7 +473,7 @@ class Testcase(Model): group_bug_information = ndb.IntegerProperty(default=0) # Fake user interaction sequences like key clicks, mouse movements, etc. - gestures = ndb.StringProperty(repeated=True, indexed=False) + gestures = StringProperty(repeated=True, indexed=False) # ASAN redzone size in bytes. redzone = ndb.IntegerProperty(default=128, indexed=False) @@ -459,37 +486,37 @@ class Testcase(Model): # Additional metadata stored as a JSON object. This should be used for # properties that are not commonly accessed and do not need to be indexed. - additional_metadata = ndb.TextProperty(indexed=False) + additional_metadata = TextProperty(indexed=False) # Boolean attribute indicating if cleanup triage needs to be done. triaged = ndb.BooleanProperty(default=False) # Project name associated with this test case. - project_name = ndb.StringProperty() + project_name = StringProperty() # keywords is used for searching. - keywords = ndb.StringProperty(repeated=True) + keywords = StringProperty(repeated=True) # Whether testcase has a bug (either bug_information or # group_bug_information). has_bug_flag = ndb.BooleanProperty() # Indices for bug_information and group_bug_information. - bug_indices = ndb.StringProperty(repeated=True) + bug_indices = StringProperty(repeated=True) # Overridden fuzzer name because actual fuzzer name can be different in many # scenarios (libfuzzer, afl, etc). - overridden_fuzzer_name = ndb.StringProperty() + overridden_fuzzer_name = StringProperty() # Platform (e.g. windows, linux, android). - platform = ndb.StringProperty() + platform = StringProperty() # Platform id (e.g. windows, linux, android:hammerhead:l). # For Android, includes device type and underlying OS version. - platform_id = ndb.StringProperty() + platform_id = StringProperty() # Impact indices for searching. - impact_indices = ndb.StringProperty(repeated=True) + impact_indices = StringProperty(repeated=True) # Whether or not a testcase is a duplicate of other testcase. is_a_duplicate_flag = ndb.BooleanProperty() @@ -501,16 +528,16 @@ class Testcase(Model): is_leader = ndb.BooleanProperty(default=False) # Fuzzer name indices - fuzzer_name_indices = ndb.StringProperty(repeated=True) + fuzzer_name_indices = StringProperty(repeated=True) # The impacted version indices (including both beta and stable). - impact_version_indices = ndb.StringProperty(repeated=True) + impact_version_indices = StringProperty(repeated=True) # The impacted stable version. - impact_stable_version = ndb.StringProperty() + impact_stable_version = StringProperty() # The impacted stable version indices. - impact_stable_version_indices = ndb.StringProperty(repeated=True) + impact_stable_version_indices = StringProperty(repeated=True) # The impacted stable version is merely probable (not definite). Because # for a non-asan build, we don't have a stable/beta build. Therefore, we @@ -518,10 +545,10 @@ class Testcase(Model): impact_stable_version_likely = ndb.BooleanProperty() # The impacted beta version. - impact_beta_version = ndb.StringProperty() + impact_beta_version = StringProperty() # The impacted beta version indices. - impact_beta_version_indices = ndb.StringProperty(repeated=True) + impact_beta_version_indices = StringProperty(repeated=True) # The impacted beta version is merely probable (not definite). See the # comment on impact_stable_version_likely. @@ -531,10 +558,10 @@ class Testcase(Model): is_impact_set_flag = ndb.BooleanProperty() # Code coverage data for the testcase. - coverage = ndb.StringProperty() + coverage = StringProperty() # Uploader email address. - uploader_email = ndb.StringProperty() + uploader_email = StringProperty() def has_blame(self): return self.project_name == 'chromium' @@ -674,14 +701,14 @@ class DataBundle(Model): VALID_NAME_REGEX = NAME_CHECK_REGEX # The data bundle's name (important for identifying shared bundles). - name = ndb.StringProperty() + name = StringProperty() # Name of cloud storage bucket on GCS. - bucket_name = ndb.StringProperty() + bucket_name = StringProperty() # Data bundle's source (for accountability). # TODO(ochang): Remove. - source = ndb.StringProperty() + source = StringProperty() # If data bundle can be unpacked locally or needs nfs. is_local = ndb.BooleanProperty(default=True) @@ -698,42 +725,42 @@ class DataBundle(Model): class Config(Model): """Configuration.""" - previous_hash = ndb.StringProperty(default='') + previous_hash = StringProperty(default='') # Project's url. - url = ndb.StringProperty(default='') + url = StringProperty(default='') # Issue tracker client authentication parameters. - client_credentials = ndb.TextProperty(default='') + client_credentials = TextProperty(default='') # Build apiary authentication parameters. - build_apiary_service_account_email = ndb.StringProperty(default='') - build_apiary_service_account_private_key = ndb.TextProperty(default='') + build_apiary_service_account_email = StringProperty(default='') + build_apiary_service_account_private_key = TextProperty(default='') # Google test account for login, gms testing, etc. - test_account_email = ndb.StringProperty(default='') - test_account_password = ndb.StringProperty(default='') + test_account_email = StringProperty(default='') + test_account_password = StringProperty(default='') # Privileged users. - privileged_users = ndb.TextProperty(default='') + privileged_users = TextProperty(default='') # Admin contact string. - contact_string = ndb.StringProperty(default='') + contact_string = StringProperty(default='') # Component to repository mappings. - component_repository_mappings = ndb.TextProperty(default='') + component_repository_mappings = TextProperty(default='') # URL for help page for reproducing issues. - reproduction_help_url = ndb.StringProperty(default='') + reproduction_help_url = StringProperty(default='') # Documentation url. - documentation_url = ndb.StringProperty(default='') + documentation_url = StringProperty(default='') # Bug report url. - bug_report_url = ndb.StringProperty(default='') + bug_report_url = StringProperty(default='') # Platforms that coverage is supported for. - platform_group_mappings = ndb.TextProperty(default='') + platform_group_mappings = TextProperty(default='') # More relaxed restrictions: allow CC'ed users and reporters of issues to view # testcase details. @@ -744,28 +771,28 @@ class Config(Model): relax_security_bug_restrictions = ndb.BooleanProperty(default=False) # Coverage reports bucket. - coverage_reports_bucket = ndb.StringProperty(default='') + coverage_reports_bucket = StringProperty(default='') # For GitHub API. - github_credentials = ndb.StringProperty(default='') + github_credentials = StringProperty(default='') # OAuth2 client id for the reproduce tool. - reproduce_tool_client_id = ndb.StringProperty(default='') + reproduce_tool_client_id = StringProperty(default='') # OAuth2 client secret for the reproduce tool. - reproduce_tool_client_secret = ndb.StringProperty(default='') + reproduce_tool_client_secret = StringProperty(default='') # Pub/Sub topics for the Predator service. - predator_crash_topic = ndb.StringProperty(default='') - predator_result_topic = ndb.StringProperty(default='') + predator_crash_topic = StringProperty(default='') + predator_result_topic = StringProperty(default='') # Wifi connection information. - wifi_ssid = ndb.StringProperty(default='') - wifi_password = ndb.StringProperty(default='') + wifi_ssid = StringProperty(default='') + wifi_password = StringProperty(default='') # SendGrid config. - sendgrid_api_key = ndb.StringProperty(default='') - sendgrid_sender = ndb.StringProperty(default='') + sendgrid_api_key = StringProperty(default='') + sendgrid_sender = StringProperty(default='') class TestcaseUploadMetadata(Model): @@ -774,16 +801,16 @@ class TestcaseUploadMetadata(Model): timestamp = ndb.DateTimeProperty() # Testcase filename. - filename = ndb.StringProperty() + filename = StringProperty() # Current status of the testcase. - status = ndb.StringProperty() + status = StringProperty() # Uploader email address. - uploader_email = ndb.StringProperty() + uploader_email = StringProperty() # Name of the bot that ran analyze on this testcase. - bot_name = ndb.StringProperty() + bot_name = StringProperty() # Id of the associated testcase. testcase_id = ndb.IntegerProperty() @@ -792,7 +819,7 @@ class TestcaseUploadMetadata(Model): duplicate_of = ndb.IntegerProperty() # Blobstore key for the testcase associated with this object. - blobstore_key = ndb.StringProperty() + blobstore_key = StringProperty() # Testcase timeout. timeout = ndb.IntegerProperty() @@ -801,10 +828,10 @@ class TestcaseUploadMetadata(Model): bundled = ndb.BooleanProperty() # Path to the file in the archive. - path_in_archive = ndb.TextProperty() + path_in_archive = TextProperty() # Original blobstore key for this object (used for archives). - original_blobstore_key = ndb.StringProperty() + original_blobstore_key = StringProperty() # Security flag. security_flag = ndb.BooleanProperty(default=False) @@ -821,10 +848,10 @@ class TestcaseUploadMetadata(Model): class JobTemplate(Model): # Job template name. - name = ndb.StringProperty() + name = StringProperty() # Environment string. - environment_string = ndb.TextProperty() + environment_string = TextProperty() class Job(Model): @@ -833,31 +860,31 @@ class Job(Model): VALID_NAME_REGEX = NAME_CHECK_REGEX # Job type name. - name = ndb.StringProperty() + name = StringProperty() # Job environment string. - environment_string = ndb.TextProperty() + environment_string = TextProperty() # The platform that this job can run on. - platform = ndb.StringProperty() + platform = StringProperty() # Blobstore key of the custom binary for this job. - custom_binary_key = ndb.StringProperty() + custom_binary_key = StringProperty() # Filename for the custom binary. - custom_binary_filename = ndb.StringProperty() + custom_binary_filename = StringProperty() # Revision of the custom binary. custom_binary_revision = ndb.IntegerProperty() # Description of the job. - description = ndb.TextProperty() + description = TextProperty() # Template to use, if any. - templates = ndb.StringProperty(repeated=True) + templates = StringProperty(repeated=True) # Project name. - project = ndb.StringProperty() + project = StringProperty() def get_environment(self): """Get the environment as a dict for this job, including any environment @@ -901,31 +928,31 @@ def _pre_put_hook(self): class CSRFToken(Model): """Token used to prevent CSRF attacks.""" # Value of this token. - value = ndb.StringProperty() + value = StringProperty() # Expiration time for this token. expiration_time = ndb.DateTimeProperty() # User that this token is associated with. - user_email = ndb.StringProperty() + user_email = StringProperty() class Heartbeat(Model): """Bot health metadata.""" # Name of the bot. - bot_name = ndb.StringProperty() + bot_name = StringProperty() # Time of the last heartbeat. last_beat_time = ndb.DateTimeProperty() # Task payload containing information on current task execution. - task_payload = ndb.StringProperty() + task_payload = StringProperty() # Expected end time for task. task_end_time = ndb.DateTimeProperty() # Source version (for accountability). - source_version = ndb.StringProperty() + source_version = StringProperty() class Notification(Model): @@ -934,68 +961,68 @@ class Notification(Model): testcase_id = ndb.IntegerProperty() # User that this notification was sent to. - user_email = ndb.StringProperty() + user_email = StringProperty() class BundledArchiveMetadata(Model): """Metadata needed for multiple test cases uploaded in an archive.""" # Blobstore key of the archive. - blobstore_key = ndb.StringProperty() + blobstore_key = StringProperty() # Timeout in seconds for each testcase in the bundle. timeout = ndb.IntegerProperty() # Job queue for the analyze tasks created for this bundle. - job_queue = ndb.StringProperty() + job_queue = StringProperty() # Job type that should be used for all testcases in this bundle. - job_type = ndb.StringProperty() + job_type = StringProperty() # Flag indicating whether or not these testcases need http. http_flag = ndb.BooleanProperty() # File name of the uploaded archive. - archive_filename = ndb.StringProperty() + archive_filename = StringProperty() # Email address of the uploader of the archive. - uploader_email = ndb.StringProperty() + uploader_email = StringProperty() # Fake user interaction sequences like key clicks, mouse movements, etc. - gestures = ndb.StringProperty(repeated=True) + gestures = StringProperty(repeated=True) # Optional. Revision that we discovered the crash in. crash_revision = ndb.IntegerProperty() # Optional. Additional arguments. - additional_arguments = ndb.StringProperty() + additional_arguments = StringProperty() # Optional. Bug information. - bug_information = ndb.StringProperty() + bug_information = StringProperty() # Optional. Platform id, e.g. android:shamu. - platform_id = ndb.StringProperty() + platform_id = StringProperty() # Optional. App launch command. e.g. shell am start ... - app_launch_command = ndb.StringProperty() + app_launch_command = StringProperty() # Fuzzer name. - fuzzer_name = ndb.StringProperty() + fuzzer_name = StringProperty() # Overridden fuzzer name because actual fuzzer name can be different in many # scenarios (libfuzzer, afl, etc). - overridden_fuzzer_name = ndb.StringProperty() + overridden_fuzzer_name = StringProperty() # Binary name for fuzz target (only applicable to libFuzzer, AFL). - fuzzer_binary_name = ndb.StringProperty() + fuzzer_binary_name = StringProperty() class TaskStatus(Model): """Information about task status.""" # Bot name. - bot_name = ndb.StringProperty() + bot_name = StringProperty() # Status. - status = ndb.StringProperty() + status = StringProperty() # Time of creation or last update time. time = ndb.DateTimeProperty() @@ -1004,7 +1031,7 @@ class TaskStatus(Model): class BuildMetadata(Model): """Metadata associated with a particular archived build.""" # Job type that this build belongs to. - job_type = ndb.StringProperty() + job_type = StringProperty() # Revision of the build. revision = ndb.IntegerProperty() @@ -1013,13 +1040,13 @@ class BuildMetadata(Model): bad_build = ndb.BooleanProperty(default=False) # Stdout and stderr. - console_output = ndb.TextProperty() + console_output = TextProperty() # Bot name. - bot_name = ndb.StringProperty() + bot_name = StringProperty() # Symbol data. - symbols = ndb.StringProperty() + symbols = StringProperty() # Creation timestamp. timestamp = ndb.DateTimeProperty() @@ -1028,7 +1055,7 @@ class BuildMetadata(Model): class ReportMetadata(Model): """Metadata associated with a crash report.""" # Job type from testcase. - job_type = ndb.StringProperty() + job_type = StringProperty() # Revision of build from report. crash_revision = ndb.IntegerProperty(default=-1) @@ -1037,28 +1064,28 @@ class ReportMetadata(Model): is_uploaded = ndb.BooleanProperty(default=False) # Product. - product = ndb.StringProperty(default='') + product = StringProperty(default='') # Version. - version = ndb.StringProperty(default='', indexed=False) + version = StringProperty(default='', indexed=False) # Key to minidump previously written to blobstore. - minidump_key = ndb.StringProperty(default='', indexed=False) + minidump_key = StringProperty(default='', indexed=False) # Processed crash bytes. serialized_crash_stack_frames = ndb.BlobProperty(default='', indexed=False) # Id of the associated testcase. - testcase_id = ndb.StringProperty(default='') + testcase_id = StringProperty(default='') # Id of the associated bot. - bot_id = ndb.StringProperty(default='', indexed=False) + bot_id = StringProperty(default='', indexed=False) # Optional upload params, stored as a JSON object. - optional_params = ndb.TextProperty(indexed=False) + optional_params = TextProperty(indexed=False) # Report id from crash/. - crash_report_id = ndb.StringProperty() + crash_report_id = StringProperty() class Lock(Model): @@ -1067,19 +1094,19 @@ class Lock(Model): expiration_time = ndb.DateTimeProperty() # The bot name denoting the holder of the lock. - holder = ndb.StringProperty() + holder = StringProperty() class FuzzTarget(Model): """Fuzz target.""" # The engine this target is a child of. - engine = ndb.StringProperty() + engine = StringProperty() # Project name. - project = ndb.StringProperty() + project = StringProperty() # Binary name. - binary = ndb.StringProperty() + binary = StringProperty() def _pre_put_hook(self): """Pre-put hook.""" @@ -1134,13 +1161,13 @@ class FuzzTargetJob(Model): """Mapping between fuzz target and jobs with additional metadata for selection.""" # Fully qualified fuzz target name. - fuzz_target_name = ndb.StringProperty() + fuzz_target_name = StringProperty() # Job this target ran as. - job = ndb.StringProperty() + job = StringProperty() # Engine this ran as. - engine = ndb.StringProperty() + engine = StringProperty() # Relative frequency with which to select this fuzzer. weight = ndb.FloatProperty(default=1.0) @@ -1158,9 +1185,9 @@ class FuzzStrategyProbability(Model): """Mapping between fuzz strategies and probabilities with which they should be selected.""" - strategy_name = ndb.StringProperty() + strategy_name = StringProperty() probability = ndb.FloatProperty() - engine = ndb.StringProperty() + engine = StringProperty() def fuzz_target_job_key(fuzz_target_name, job): @@ -1171,13 +1198,13 @@ def fuzz_target_job_key(fuzz_target_name, job): class ExternalUserPermission(Model): """Permissions for external users.""" # Email user is authenticated as. - email = ndb.StringProperty() + email = StringProperty() # Type of |entity_name|. Can be one of the values of PermissionEntityKind. entity_kind = ndb.IntegerProperty() # Name of the entity that user is allowed to view. - entity_name = ndb.StringProperty() + entity_name = StringProperty() # Whether or not |allowed_name| is a prefix. is_prefix = ndb.BooleanProperty(default=False) @@ -1201,22 +1228,22 @@ class FiledBug(Model): group_id = ndb.IntegerProperty() # Crash type for easy reference. - crash_type = ndb.StringProperty() + crash_type = StringProperty() # Crash state for easy reference. - crash_state = ndb.StringProperty() + crash_state = StringProperty() # Is it a security bug? security_flag = ndb.BooleanProperty() # Platform id. - platform_id = ndb.StringProperty() + platform_id = StringProperty() class CoverageInformation(Model): """Coverage info.""" date = ndb.DateProperty(auto_now_add=True) - fuzzer = ndb.StringProperty() + fuzzer = StringProperty() # Function coverage information. functions_covered = ndb.IntegerProperty() @@ -1229,18 +1256,18 @@ class CoverageInformation(Model): # Corpus size information. corpus_size_units = ndb.IntegerProperty() corpus_size_bytes = ndb.IntegerProperty() - corpus_location = ndb.StringProperty() + corpus_location = StringProperty() # Corpus backup information. - corpus_backup_location = ndb.StringProperty() + corpus_backup_location = StringProperty() # Quarantine size information. quarantine_size_units = ndb.IntegerProperty() quarantine_size_bytes = ndb.IntegerProperty() - quarantine_location = ndb.StringProperty() + quarantine_location = StringProperty() # Link to the HTML report. - html_report_url = ndb.StringProperty() + html_report_url = StringProperty() def _pre_put_hook(self): """Pre-put hook.""" @@ -1262,20 +1289,20 @@ def coverage_information_key(project_qualified_fuzzer_name, date): class Trial(Model): """Trials for specific binaries.""" # App name that this trial is applied to. E.g. "d8" or "chrome". - app_name = ndb.StringProperty() + app_name = StringProperty() # Chance to select this set of arguments. Zero to one. probability = ndb.FloatProperty() # Additional arguments to apply if selected. - app_args = ndb.StringProperty(indexed=False) + app_args = StringProperty(indexed=False) # TODO(ochang): Make this generic. class OssFuzzProject(Model): """Represents a project that has been set up for OSS-Fuzz.""" # Name of the project. - name = ndb.StringProperty() + name = StringProperty() # Whether or not the project should run on high end hosts. high_end = ndb.BooleanProperty(default=False) @@ -1287,10 +1314,10 @@ class OssFuzzProject(Model): disk_size_gb = ndb.IntegerProperty() # Service account for this project. - service_account = ndb.StringProperty() + service_account = StringProperty() # CCs for the project. - ccs = ndb.StringProperty(repeated=True) + ccs = StringProperty(repeated=True) class OssFuzzProjectInfo(Model): @@ -1300,16 +1327,16 @@ class OssFuzzProjectInfo(Model): class ClusterInfo(Model): """Cpu allocation information for a project in a zone.""" # The cluster for the CPU allocation. - cluster = ndb.StringProperty() + cluster = StringProperty() # The number of allocated CPUs in this cluster. cpu_count = ndb.IntegerProperty(default=0) # The GCE zone for this cluster. - gce_zone = ndb.StringProperty() + gce_zone = StringProperty() # Name of the project. - name = ndb.StringProperty() + name = StringProperty() # Information about CPUs in each cluster. clusters = ndb.StructuredProperty(ClusterInfo, repeated=True) @@ -1321,22 +1348,22 @@ def get_cluster_info(self, name): class HostWorkerAssignment(Model): """Host worker assignment information.""" # The host instance name. - host_name = ndb.StringProperty() + host_name = StringProperty() # The instance number (0 to WORKERS_PER_HOST - 1). instance_num = ndb.IntegerProperty() # The worker instance name. - worker_name = ndb.StringProperty() + worker_name = StringProperty() # The project name. - project_name = ndb.StringProperty() + project_name = StringProperty() class WorkerTlsCert(Model): """TLS certs for untrusted workers.""" # The name of the project. - project_name = ndb.StringProperty() + project_name = StringProperty() # The contents of the TLS cert. cert_contents = ndb.BlobProperty() @@ -1347,9 +1374,9 @@ class WorkerTlsCert(Model): class FuzzerJob(Model): """Mapping between a fuzzer and job with additional metadata for selection.""" - fuzzer = ndb.StringProperty() - job = ndb.StringProperty() - platform = ndb.StringProperty() + fuzzer = StringProperty() + job = StringProperty() + platform = StringProperty() weight = ndb.FloatProperty(default=1.0) multiplier = ndb.FloatProperty(default=1.0) @@ -1362,10 +1389,10 @@ def actual_weight(self): class OssFuzzBuildFailure(Model): """Represents build failure.""" # Project name. - project_name = ndb.StringProperty() + project_name = StringProperty() # The monorail issue ID for the failure. - issue_id = ndb.StringProperty() + issue_id = StringProperty() # The last timestamp of the build. last_checked_timestamp = ndb.DateTimeProperty() @@ -1374,12 +1401,12 @@ class OssFuzzBuildFailure(Model): consecutive_failures = ndb.IntegerProperty(default=0) # Build type (fuzzing, coverage, etc). - build_type = ndb.StringProperty() + build_type = StringProperty() class Admin(Model): """Records an admin user.""" - email = ndb.StringProperty() + email = StringProperty() class TestcaseVariant(Model): @@ -1392,16 +1419,16 @@ class TestcaseVariant(Model): status = ndb.IntegerProperty(default=0) # Job type for the testcase variant. - job_type = ndb.StringProperty() + job_type = StringProperty() # Revision that the testcase variant was tried against. revision = ndb.IntegerProperty() # Crash type. - crash_type = ndb.StringProperty() + crash_type = StringProperty() # Crash state. - crash_state = ndb.StringProperty() + crash_state = StringProperty() # Bool to indicate if it is a security bug? security_flag = ndb.BooleanProperty() @@ -1411,4 +1438,4 @@ class TestcaseVariant(Model): # Similar testcase reproducer key (optional). This is set in case we notice a # similar crash on another platform. - reproducer_key = ndb.StringProperty() + reproducer_key = StringProperty()