diff --git a/cli/helpers.py b/cli/helpers.py index b717ab295..cdd2986e9 100644 --- a/cli/helpers.py +++ b/cli/helpers.py @@ -167,6 +167,8 @@ def get_item_yaml_values( if values: if isinstance(values, list): values_set = set(values) + elif isinstance(values, dict): + values_set = values else: values_set.add(values) values_dict[key] = values_set diff --git a/cli/item_to_function.py b/cli/item_to_function.py index 87f622d47..a1bb4b168 100644 --- a/cli/item_to_function.py +++ b/cli/item_to_function.py @@ -55,17 +55,17 @@ help="If -b/--bump_version is enabled, increase the minor version in the item.yaml file", ) def item_to_function_cli( - item_path: str, output_path: Optional[str], code_output: bool, format_code: bool, bump_version: bool + item_path: str, output_path: Optional[str], code_output: bool, format_code: bool, bump_version: bool ): item_to_function(item_path, output_path, code_output, format_code, bump_version) def item_to_function( - item_path: str, - output_path: Optional[str] = None, - code_output: bool = False, - format_code: bool = True, - bump_version: bool = False, + item_path: str, + output_path: Optional[str] = None, + code_output: bool = False, + format_code: bool = True, + bump_version: bool = False, ): item_path = Path(item_path) if item_path.is_dir(): @@ -78,9 +78,9 @@ def item_to_function( # That means we need to search for items inside this direcotry else: for inner_dir in PathIterator( - root=item_path.parent, - rule=is_item_dir, - as_path=True, + root=item_path.parent, + rule=is_item_dir, + as_path=True, ): try: _output_path = output_path or (inner_dir / "function.yaml") @@ -119,11 +119,11 @@ def _get_item_yaml(item_path: Path) -> dict: def create_function_yaml( - item_path: Union[str, Path], - output_path: Optional[str] = None, - code_output: bool = False, - format_code: bool = True, - bump_version: bool = False, + item_path: Union[str, Path], + output_path: Optional[str] = None, + code_output: bool = False, + format_code: bool = True, + bump_version: bool = False, ): item_path = Path(item_path) if bump_version: @@ -161,7 +161,8 @@ def create_function_yaml( # remove build info from object function_object.spec.build.code_origin = '' function_object.spec.build.origin_filename = '' - function_object.spec.state_thresholds=None + if 'state_thresholds' not in spec: + function_object.spec.state_thresholds = None custom_fields = spec.get("customFields", {}) for key, value in custom_fields.items(): diff --git a/pii_recognizer/function.yaml b/pii_recognizer/function.yaml index 086bc3867..54b448d9c 100644 --- a/pii_recognizer/function.yaml +++ b/pii_recognizer/function.yaml @@ -2,8 +2,8 @@ kind: job metadata: name: pii-recognizer tag: '' - hash: 0972dbbfd83e86970a3655774ace0c074ea617ce - project: llm-workflow-gilads + hash: b09b7b9a4ffd55088d665a0191055411e9198a2f + project: '' labels: author: pgw categories: @@ -14,48 +14,67 @@ spec: args: [] image: '' build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKCmltcG9ydCB3YXJuaW5ncwppbXBvcnQgb3MKaW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG1scnVuCmltcG9ydCBwYXRobGliCmltcG9ydCB0ZW1wZmlsZQppbXBvcnQgbmx0awpmcm9tIHRxZG0uYXV0byBpbXBvcnQgdHFkbQpmcm9tIHR5cGluZyBpbXBvcnQgTGlzdCwgVHVwbGUsIFNldCwgT3B0aW9uYWwsIERpY3QsIEFueSwgVW5pb24KZnJvbSBjb2xsZWN0aW9ucy5hYmMgaW1wb3J0IEl0ZXJhYmxlCmltcG9ydCBwcmVzaWRpb19hbmFseXplciBhcyBwYQppbXBvcnQgcHJlc2lkaW9fYW5vbnltaXplciBhcyBwcmVfYW5veW1pemVyCmZyb20gcHJlc2lkaW9fYW5vbnltaXplci5lbnRpdGllcyBpbXBvcnQgT3BlcmF0b3JDb25maWcKaW1wb3J0IGFubm90YXRlZF90ZXh0LnV0aWwgYXMgYXRfdXRpbAoKdHJ5OgogICAgaW1wb3J0IGZsYWlyIGFzIGZsCmV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgcHJpbnQoIkZsYWlyIGlzIG5vdCBpbnN0YWxsZWQiKQoKIyBUaGVyZSBpcyBhIGNvbmZsaWN0IGJldHdlZW4gUnVzdC1iYXNlZCB0b2tlbml6ZXJzJyBwYXJhbGxlbCBwcm9jZXNzaW5nCiMgYW5kIFB5dGhvbidzIGZvcmsgb3BlcmF0aW9ucyBkdXJpbmcgbXVsdGlwcm9jZXNzaW5nLiBUbyBhdm9pZCB0aGlzLCB3ZSBuZWVkCiMgdGhlIGZvbGxvd2luZyB0d28gbGluZXMKCm9zLmVudmlyb25bIlRPS0VOSVpFUlNfUEFSQUxMRUxJU00iXSA9ICJmYWxzZSIKd2FybmluZ3MuZmlsdGVyd2FybmluZ3MoImlnbm9yZSIpCgpsb2dnZXIgPSBsb2dnaW5nLmdldExvZ2dlcigicGlpLXJlY29nbml6ZXIiKQoKCiMgQWRkIHRoZSBjb25zdGFudCBjbGFzc2VzIG9mIE1vZGVscyBhbmQgRW50aXRpZXMgdG8gZ292ZXJuIHRoZSB3aG9sZSBwYWNrYWdlCmNsYXNzIE1vZGVsczoKICAgIFdIT0xFID0gIndob2xlIgogICAgUEFUVEVSTiA9ICJwYXR0ZXJuIgogICAgU1BBQ1kgPSAic3BhY3kiCiAgICBGTEFJUiA9ICJmbGFpciIKCgpjbGFzcyBFbnRpdGllczoKICAgIENSRURJVF9DQVJEID0gIkNSRURJVF9DQVJEIgogICAgU1NOID0gIlNTTiIKICAgIFBIT05FID0gIlBIT05FIgogICAgRU1BSUwgPSAiRU1BSUwiCiAgICBMT0NBVElPTiA9ICJMT0NBVElPTiIKICAgIFBFUlNPTiA9ICJQRVJTT04iCiAgICBOUlAgPSAiTlJQIgogICAgT1JHQU5JWkFUSU9OID0gIk9SR0FOSVpBVElPTiIKICAgIERBVEVfVElNRSA9ICJEQVRFX1RJTUUiCiAgICBHUEUgPSAoIkdQRSIsKQogICAgTUFDX0FERFJFU1MgPSAiTUFDX0FERFJFU1MiCiAgICBVU19CQU5LX05VTUJFUiA9ICJVU19CQU5LX05VTUJFUiIKICAgIElNRUkgPSAiSU1FSSIKICAgIFRJVExFID0gIlRJVExFIgogICAgTElDRU5TRV9QTEFURSA9ICJMSUNFTlNFX1BMQVRFIgogICAgVVNfUEFTU1BPUlQgPSAiVVNfUEFTU1BPUlQiCiAgICBDVVJSRU5DWSA9ICJDVVJSRU5DWSIKICAgIFJPVVRJTkdfTlVNQkVSID0gIlJPVVRJTkdfTlVNQkVSIgogICAgVVNfSVRJTiA9ICJVU19JVElOIgogICAgVVNfQkFOS19OVU1CRVIgPSAiVVNfQkFOS19OVU1CRVIiCiAgICBVU19EUklWRVJfTElDRU5TRSA9ICJVU19EUklWRVJfTElDRU5TRSIKICAgIEFHRSA9ICJBR0UiCiAgICBQQVNTV09SRCA9ICJQQVNTV09SRCIKICAgIFNXSUZUX0NPREUgPSAiU1dJRlRfQ09ERSIKCgpjbGFzcyBQYXR0ZXJuUmVjb2duaXplckZhY3Rvcnk6CiAgICAiIiIKICAgIEZhY3RvcnkgZm9yIGNyZWF0aW5nIHBhdHRlcm4gcmVjb2duaXplcnMsIGl0IGNhbiBiZSBleHRlbmRlZCBpbiB0aGUgZnV0dXJlIHRvCiAgICBhZGQgbW9yZSByZWdleCBwYXR0ZXJuIGZvciBkaWZmZXJlbnQgZW50aXRpZXMuIEZvciB0aGUgcGF0dGVybiByZWNvZ25pemVyIHRvIHdvcmssCiAgICB3ZSBuZWVkIGNvbnN0cnVjdCBhIGxpc3Qgb2YgcmVnZXggcGF0dGVybnMgZm9yIGVhY2ggZW50aXR5LgogICAgIiIiCgogICAgUkVDT0dOSVpBQkxFX0VOVElUSUVTID0gewogICAgICAgICJDUkVESVRfQ0FSRCI6IFtwYS5QYXR0ZXJuKCJDUkVESVRfQ0FSRCIsIHIiXGIoPzpcZFsgLV0qPyl7MTMsMTZ9XGIiLCAwLjUpXSwKICAgICAgICAiU1NOIjogW3BhLlBhdHRlcm4oIlNTTiIsIHIiXGJcZHszfS0/XGR7Mn0tP1xkezR9XGIiLCAwLjUpXSwKICAgICAgICAiUEhPTkUiOiBbcGEuUGF0dGVybigiUEhPTkUiLCByIlwoP1xkezN9XCk/Wy0uXHNdP1xkezN9Wy0uXHNdP1xkezR9IiwgMC41KV0sCiAgICAgICAgIkVNQUlMIjogW3BhLlBhdHRlcm4oIkVNQUlMIiwgciJcUytAXFMrIiwgMC41KV0sCiAgICB9CgogICAgIyBjcmVhdGUgYSBsaXN0IG9mIHBhdHRlcm4gcmVjb2duaXplcnMKICAgIEBjbGFzc21ldGhvZAogICAgZGVmIF9jcmVhdGVfcGF0dGVybl9yZWNvZ25pemVyKGNscyk6CiAgICAgICAgIiIiCiAgICAgICAgRm9yIGVhY2ggZW50aXR5LCBjcmVhdGUgYSBsaXN0IG9mIHBhdHRlcm5zIHRvIHJlY29nbml6ZSBpdAoKICAgICAgICA6cGFyYW0gY2xzOiBQYXR0ZXJuUmVjb2duaXplckZhY3RvcnkgY2xhc3MKCiAgICAgICAgOnJldHVybnM6IExpc3Qgb2YgcGF0dGVybiByZWNvZ25pemVycwogICAgICAgICIiIgoKICAgICAgICAjIEVudGl0aWVzIHRvIHJlY29nbml6ZSBhbmQgdGhlaXIgcmVnZXggcGF0dGVybnMKCiAgICAgICAgcmV0dXJuIFsKICAgICAgICAgICAgcGEuUGF0dGVyblJlY29nbml6ZXIoc3VwcG9ydGVkX2VudGl0eT1lbnRpdHksIHBhdHRlcm5zPXBhdHRlcm4pCiAgICAgICAgICAgIGZvciBlbnRpdHksIHBhdHRlcm4gaW4gY2xzLlJFQ09HTklaQUJMRV9FTlRJVElFUy5pdGVtcygpCiAgICAgICAgXQoKCmNsYXNzIEN1c3RvbVNwYWN5UmVjb2duaXplcihwYS5Mb2NhbFJlY29nbml6ZXIpOgogICAgIiIiCiAgICBDdXN0b20gU3BhY3kgUmVjb2duaXplciBmcm9tIFByZXNpZGlvIEFuYWx5emVyIHRyYWluZWQgb24gUHJpdnkgZGF0YS4KICAgIFRoZSBwcml2eSBkYXRhIGlzIGdlbmVyYXRlZCB1c2luZyB0aGlzIGh0dHBzOi8vZ2l0aHViLmNvbS9waXhpZS1pby9waXhpZS90cmVlL21haW4vc3JjL2RhdGFnZW4vcGlpL3ByaXZ5CiAgICBJdCBjYW4gYmUgdXNlZCB0byByZWNvZ25pemUgY3VzdG9tIGVudGl0aWVzLCBTaW5jZSB3ZSB3YW50IHRvIHVzZSBQcmVzaWRpbydzIFJlZ2lzdHJpZXMgdG8gZ2VuZXJhdGUgQW5hbHl6ZXJFbmdpbmUsCiAgICBpdCBpbmhlcml0cyBmcm9tIFByZXNpZGlvIEFuYWx5emVyJ3MgTG9jYWxSZWNvZ25pemVyIGNsYXNzLgogICAgIiIiCgogICAgIyBFbnRpdGllcyB0byByZWNvZ25pemUKCiAgICBSRUNPR05JWkFCTEVfRU5USVRJRVMgPSB7CiAgICAgICAgIkxPQ0FUSU9OIiwKICAgICAgICAiUEVSU09OIiwKICAgICAgICAiTlJQIiwKICAgICAgICAiT1JHQU5JWkFUSU9OIiwKICAgICAgICAiREFURV9USU1FIiwKICAgIH0KCiAgICAjIERlZmF1bHQgZXhwbGFuYXRpb24gZm9yIHRoaXMgcmVjb2duaXplcgoKICAgIF9ERUZBVUxUX0VYUExBTkFUSU9OID0gKAogICAgICAgICJJZGVudGlmaWVkIGFzIHt9IGJ5IFNwYWN5J3MgTmFtZWQgRW50aXR5IFJlY29nbml0aW9uIChQcml2eS10cmFpbmVkKSIKICAgICkKCiAgICAjIExhYmVsIGdyb3VwcyB0byBjaGVjawoKICAgIF9ERUZBVUxUX0NIRUNLX0xBQkVMX0dST1VQUyA9IFsKICAgICAgICAoeyJMT0NBVElPTiJ9LCB7IkxPQyIsICJMT0NBVElPTiIsICJTVFJFRVRfQUREUkVTUyIsICJDT09SRElOQVRFIn0pLAogICAgICAgICh7IlBFUlNPTiJ9LCB7IlBFUiIsICJQRVJTT04ifSksCiAgICAgICAgKHsiTlJQIn0sIHsiTk9SUCIsICJOUlAifSksCiAgICAgICAgKHsiT1JHQU5JWkFUSU9OIn0sIHsiT1JHIn0pLAogICAgICAgICh7IkRBVEVfVElNRSJ9LCB7IkRBVEVfVElNRSJ9KSwKICAgIF0KCiAgICAjIHByZXRyYWluZWQgbW9kZWwgZm9yIHRoaXMgcmVjb2duaXplcgoKICAgIF9ERUZBVUxUX01PREVMX0xBTkdVQUdFUyA9IHsKICAgICAgICAiZW4iOiAiYmVraS9lbl9zcGFjeV9waWlfZGlzdGlsYmVydCIsCiAgICB9CgogICAgX0RFRkFVTFRfUFJFU0lESU9fRVFVSVZBTEVOQ0VTID0gewogICAgICAgICJQRVIiOiAiUEVSU09OIiwKICAgICAgICAiTE9DIjogIkxPQ0FUSU9OIiwKICAgICAgICAiT1JHIjogIk9SR0FOSVpBVElPTiIsCiAgICAgICAgIk5ST1AiOiAiTlJQIiwKICAgICAgICAiREFURV9USU1FIjogIkRBVEVfVElNRSIsCiAgICB9CgogICAgZGVmIF9faW5pdF9fKAogICAgICAgIHNlbGYsCiAgICAgICAgc3VwcG9ydGVkX2xhbmd1YWdlOiBzdHIgPSAiZW4iLAogICAgICAgIHN1cHBvcnRlZF9lbnRpdGllczogTGlzdFtzdHJdID0gTm9uZSwKICAgICAgICBjaGVja19sYWJlbF9ncm91cHM6IFR1cGxlW1NldCwgU2V0XSA9IE5vbmUsCiAgICAgICAgY29udGV4dDogTGlzdFtzdHJdID0gTm9uZSwKICAgICAgICBuZXJfc3RyZW5ndGg6IGZsb2F0ID0gMSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgSW5pdGlhbGl6ZSBTcGFjeSBSZWNvZ25pemVyLgoKICAgICAgICA6cGFyYW0gc3VwcG9ydGVkX2xhbmd1YWdlOiBMYW5ndWFnZSB0byB1c2UsIGRlZmF1bHQgaXMgRW5nbGlzaAogICAgICAgIDpwYXJhbSBzdXBwb3J0ZWRfZW50aXRpZXM6IEVudGl0aWVzIHRvIHVzZSBmb3IgcmVjb2duaXRpb24KICAgICAgICA6cGFyYW0gY2hlY2tfbGFiZWxfZ3JvdXBzOiBMYWJlbCBncm91cHMgdG8gY2hlY2sgZm9yIHRoZSBlbnRpdGllcwogICAgICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgIENvbnRleHQgdG8gdXNlIGlmIGFueQogICAgICAgIDpwYXJhbSBuZXJfc3RyZW5ndGg6ICAgICAgIERlZmF1bHQgY29uZmlkZW5jZSBmb3IgTkVSIHByZWRpY3Rpb24KCiAgICAgICAgOnJldHVybnM6IFNwYWN5UmVjb2duaXplciBvYmplY3QKICAgICAgICAiIiIKCiAgICAgICAgIyBEZWZhdWx0IGNvbmZpZGVuY2UgZm9yIE5FUiBwcmVkaWN0aW9uCiAgICAgICAgc2VsZi5uZXJfc3RyZW5ndGggPSBuZXJfc3RyZW5ndGgKCiAgICAgICAgc2VsZi5jaGVja19sYWJlbF9ncm91cHMgPSBjaGVja19sYWJlbF9ncm91cHMgb3Igc2VsZi5fREVGQVVMVF9DSEVDS19MQUJFTF9HUk9VUFMKICAgICAgICBzdXBwb3J0ZWRfZW50aXRpZXMgPSBzdXBwb3J0ZWRfZW50aXRpZXMgb3Igc2VsZi5SRUNPR05JWkFCTEVfRU5USVRJRVMKICAgICAgICBzdXBlcigpLl9faW5pdF9fKAogICAgICAgICAgICBzdXBwb3J0ZWRfZW50aXRpZXM9c3VwcG9ydGVkX2VudGl0aWVzLAogICAgICAgICAgICBzdXBwb3J0ZWRfbGFuZ3VhZ2U9c3VwcG9ydGVkX2xhbmd1YWdlLAogICAgICAgICkKCiAgICAjIGdldCB0aGUgcHJlc2lkaW8gZXhwbGFuYXRpb24gZm9yIHRoZSByZXN1bHQKCiAgICBkZWYgX2J1aWxkX3NwYWN5X2V4cGxhbmF0aW9uKAogICAgICAgIHNlbGYsIG9yaWdpbmFsX3Njb3JlOiBmbG9hdCwgZXhwbGFuYXRpb246IHN0cgogICAgKSAtPiBwYS5BbmFseXNpc0V4cGxhbmF0aW9uOgogICAgICAgICIiIgogICAgICAgIENyZWF0ZSBleHBsYW5hdGlvbiBmb3Igd2h5IHRoaXMgcmVzdWx0IHdhcyBkZXRlY3RlZC4KCiAgICAgICAgOnBhcmFtIG9yaWdpbmFsX3Njb3JlOiBTY29yZSBnaXZlbiBieSB0aGlzIHJlY29nbml6ZXIKICAgICAgICA6cGFyYW0gZXhwbGFuYXRpb246ICAgIEV4cGxhbmF0aW9uIHN0cmluZwoKICAgICAgICA6cmV0dXJuczogUHJlc2lkaW8gQW5hbHlzaXNFeHBsYW5hdGlvbiBvYmplY3QKICAgICAgICAiIiIKICAgICAgICBleHBsYW5hdGlvbiA9IHBhLkFuYWx5c2lzRXhwbGFuYXRpb24oCiAgICAgICAgICAgIHJlY29nbml6ZXI9c2VsZi5fX2NsYXNzX18uX19uYW1lX18sCiAgICAgICAgICAgIG9yaWdpbmFsX3Njb3JlPW9yaWdpbmFsX3Njb3JlLAogICAgICAgICAgICB0ZXh0dWFsX2V4cGxhbmF0aW9uPWV4cGxhbmF0aW9uLAogICAgICAgICkKICAgICAgICByZXR1cm4gZXhwbGFuYXRpb24KCiAgICAjIG1haW4gbWV0aG9kIGZvciB0aGUgcmVjb2duaXplcgogICAgZGVmIGFuYWx5emUoc2VsZiwgdGV4dDogc3RyLCBlbnRpdGllczogTGlzdFtzdHJdLCBubHBfYXJ0aWZhY3RzPU5vbmUpOiAgIyBub3FhIEQxMDIKICAgICAgICAiIiIKICAgICAgICBBbmFseXplIHRleHQgdXNpbmcgU3BhY3kuCgogICAgICAgIDpwYXJhbSB0ZXh0OiAgICAgICAgICBUZXh0IHRvIGFuYWx5emUKICAgICAgICA6cGFyYW0gZW50aXRpZXM6ICAgICAgRW50aXRpZXMgdG8gYW5hbHl6ZQogICAgICAgIDpwYXJhbSBubHBfYXJ0aWZhY3RzOiBOTFAgYXJ0aWZhY3RzIHRvIHVzZQoKICAgICAgICA6cmV0dXJuczogTGlzdCBvZiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IG9iamVjdHMKICAgICAgICAiIiIKICAgICAgICByZXN1bHRzID0gW10KICAgICAgICBpZiBub3QgbmxwX2FydGlmYWN0czoKICAgICAgICAgICAgbG9nZ2VyLndhcm5pbmcoIlNraXBwaW5nIFNwYUN5LCBubHAgYXJ0aWZhY3RzIG5vdCBwcm92aWRlZC4uLiIpCiAgICAgICAgICAgIHJldHVybiByZXN1bHRzCgogICAgICAgIG5lcl9lbnRpdGllcyA9IG5scF9hcnRpZmFjdHMuZW50aXRpZXMKCiAgICAgICAgIyByZWNvZ25pemUgdGhlIHN1cHBvcnRlZCBlbnRpdGllcwogICAgICAgIGZvciBlbnRpdHkgaW4gZW50aXRpZXM6CiAgICAgICAgICAgIGlmIGVudGl0eSBub3QgaW4gc2VsZi5zdXBwb3J0ZWRfZW50aXRpZXM6CiAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICBmb3IgZW50IGluIG5lcl9lbnRpdGllczoKICAgICAgICAgICAgICAgIGlmIG5vdCBzZWxmLl9fY2hlY2tfbGFiZWwoZW50aXR5LCBlbnQubGFiZWxfLCBzZWxmLmNoZWNrX2xhYmVsX2dyb3Vwcyk6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKCiAgICAgICAgICAgICAgICAjIHN0cmluZyBvZiB0aGUgZXhwbGFuYXRpb24gc2F5aW5nIHRoZSBlbnRpdHkgaXMgcmVjb2duaXplZCBieSBzcGFjeQogICAgICAgICAgICAgICAgdGV4dHVhbF9leHBsYW5hdGlvbiA9IHNlbGYuX0RFRkFVTFRfRVhQTEFOQVRJT04uZm9ybWF0KGVudC5sYWJlbF8pCiAgICAgICAgICAgICAgICBleHBsYW5hdGlvbiA9IHNlbGYuX2J1aWxkX3NwYWN5X2V4cGxhbmF0aW9uKAogICAgICAgICAgICAgICAgICAgIHNlbGYubmVyX3N0cmVuZ3RoLCB0ZXh0dWFsX2V4cGxhbmF0aW9uCiAgICAgICAgICAgICAgICApCgogICAgICAgICAgICAgICAgIyBjcmVhdGUgdGhlIHN0YW5kYXJkIHJlc3VsdCB3aXRoIHRoZSBlbnRpdHksIHN0YXJ0LCBlbmQsIHNjb3JlLCBhbmQgZXhwbGFuYXRpb24KICAgICAgICAgICAgICAgIHNwYWN5X3Jlc3VsdCA9IHBhLlJlY29nbml6ZXJSZXN1bHQoCiAgICAgICAgICAgICAgICAgICAgZW50aXR5X3R5cGU9ZW50aXR5LAogICAgICAgICAgICAgICAgICAgIHN0YXJ0PWVudC5zdGFydF9jaGFyLAogICAgICAgICAgICAgICAgICAgIGVuZD1lbnQuZW5kX2NoYXIsCiAgICAgICAgICAgICAgICAgICAgc2NvcmU9c2VsZi5uZXJfc3RyZW5ndGgsCiAgICAgICAgICAgICAgICAgICAgYW5hbHlzaXNfZXhwbGFuYXRpb249ZXhwbGFuYXRpb24sCiAgICAgICAgICAgICAgICAgICAgcmVjb2duaXRpb25fbWV0YWRhdGE9ewogICAgICAgICAgICAgICAgICAgICAgICBwYS5SZWNvZ25pemVyUmVzdWx0LlJFQ09HTklaRVJfTkFNRV9LRVk6IHNlbGYubmFtZQogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByZXN1bHRzLmFwcGVuZChzcGFjeV9yZXN1bHQpCgogICAgICAgIHJldHVybiByZXN1bHRzCgogICAgQHN0YXRpY21ldGhvZAogICAgZGVmIF9fY2hlY2tfbGFiZWwoCiAgICAgICAgZW50aXR5OiBzdHIsIGxhYmVsOiBzdHIsIGNoZWNrX2xhYmVsX2dyb3VwczogVHVwbGVbU2V0LCBTZXRdCiAgICApIC0+IGJvb2w6CiAgICAgICAgIiIiCiAgICAgICAgQ2hlY2sgaWYgdGhlIGxhYmVsIGlzIGluIHRoZSBsYWJlbCBncm91cC4KCiAgICAgICAgOnBhcmFtIGVudGl0eTogICAgICAgICAgICAgRW50aXR5IHRvIGNoZWNrCiAgICAgICAgOnBhcmFtIGxhYmVsOiAgICAgICAgICAgICAgTGFiZWwgdG8gY2hlY2sKICAgICAgICA6cGFyYW0gY2hlY2tfbGFiZWxfZ3JvdXBzOiBMYWJlbCBncm91cHMgdG8gY2hlY2sKCiAgICAgICAgOnJldHVybnM6IFRydWUgaWYgdGhlIGxhYmVsIGlzIGluIHRoZSBsYWJlbCBncm91cCwgRmFsc2Ugb3RoZXJ3aXNlCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIGFueSgKICAgICAgICAgICAgZW50aXR5IGluIGVncnAgYW5kIGxhYmVsIGluIGxncnAgZm9yIGVncnAsIGxncnAgaW4gY2hlY2tfbGFiZWxfZ3JvdXBzCiAgICAgICAgKQoKCiMgQ2xhc3MgdG8gdXNlIEZsYWlyIHdpdGggUHJlc2lkaW8gYXMgYW4gZXh0ZXJuYWwgcmVjb2duaXplci4KY2xhc3MgRmxhaXJSZWNvZ25pemVyKHBhLkVudGl0eVJlY29nbml6ZXIpOgogICAgIiIiCiAgICBXcmFwcGVyIGZvciBhIGZsYWlyIG1vZGVsLCBpZiBuZWVkZWQgdG8gYmUgdXNlZCB3aXRoaW4gUHJlc2lkaW8gQW5hbHl6ZXIuCiAgICBUaGlzIGlzIHRvIG1ha2Ugc3VyZSB0aGUgcmVjb2duaXplciBjYW4gYmUgcmVnaXN0ZXJlZCB3aXRoIFByZXNpZGlvIHJlZ2lzdHJ5LgogICAgIiIiCgogICAgUkVDT0dOSVpBQkxFX0VOVElUSUVTID0gewogICAgICAgICJMT0NBVElPTiIsCiAgICAgICAgIlBFUlNPTiIsCiAgICAgICAgIk5SUCIsCiAgICAgICAgIkdQRSIsCiAgICAgICAgIk9SR0FOSVpBVElPTiIsCiAgICAgICAgIk1BQ19BRERSRVNTIiwKICAgICAgICAiVVNfQkFOS19OVU1CRVIiLAogICAgICAgICJJTUVJIiwKICAgICAgICAiVElUTEUiLAogICAgICAgICJMSUNFTlNFX1BMQVRFIiwKICAgICAgICAiVVNfUEFTU1BPUlQiLAogICAgICAgICJDVVJSRU5DWSIsCiAgICAgICAgIlJPVVRJTkdfTlVNQkVSIiwKICAgICAgICAiVVNfSVRJTiIsCiAgICAgICAgIlVTX0JBTktfTlVNQkVSIiwKICAgICAgICAiVVNfRFJJVkVSX0xJQ0VOU0UiLAogICAgICAgICJBR0UiLAogICAgICAgICJQQVNTV09SRCIsCiAgICAgICAgIlNXSUZUX0NPREUiLAogICAgfQoKICAgICMgVGhpcyBpcyB1c2VkIHRvIGNvbnN0cnVjdCB0aGUgZXhwbGFuYXRpb24gZm9yIHRoZSByZXN1bHQKCiAgICBfREVGQVVMVF9FWFBMQU5BVElPTiA9ICJJZGVudGlmaWVkIGFzIHt9IGJ5IEZsYWlyJ3MgTmFtZWQgRW50aXR5IFJlY29nbml0aW9uIgoKICAgIF9ERUZBVUxUX0NIRUNLX0xBQkVMX0dST1VQUyA9IFsKICAgICAgICAoeyJMT0NBVElPTiJ9LCB7IkxPQyIsICJMT0NBVElPTiIsICJTVFJFRVRfQUREUkVTUyIsICJDT09SRElOQVRFIn0pLAogICAgICAgICh7IlBFUlNPTiJ9LCB7IlBFUiIsICJQRVJTT04ifSksCiAgICAgICAgKHsiTlJQIn0sIHsiTk9SUCIsICJOUlAifSksCiAgICAgICAgKHsiR1BFIn0sIHsiR1BFIn0pLAogICAgICAgICh7Ik9SR0FOSVpBVElPTiJ9LCB7Ik9SRyJ9KSwKICAgICAgICAoeyJNQUNfQUREUkVTUyJ9LCB7Ik1BQ19BRERSRVNTIn0pLAogICAgICAgICh7IlVTX0JBTktfTlVNQkVSIn0sIHsiVVNfQkFOS19OVU1CRVIifSksCiAgICAgICAgKHsiSU1FSSJ9LCB7IklNRUkifSksCiAgICAgICAgKHsiVElUTEUifSwgeyJUSVRMRSJ9KSwKICAgICAgICAoeyJMSUNFTlNFX1BMQVRFIn0sIHsiTElDRU5TRV9QTEFURSJ9KSwKICAgICAgICAoeyJVU19QQVNTUE9SVCJ9LCB7IlVTX1BBU1NQT1JUIn0pLAogICAgICAgICh7IkNVUlJFTkNZIn0sIHsiQ1VSUkVOQ1kifSksCiAgICAgICAgKHsiUk9VVElOR19OVU1CRVIifSwgeyJST1VUSU5HX05VTUJFUiJ9KSwKICAgICAgICAoeyJBR0UifSwgeyJBR0UifSksCiAgICAgICAgKHsiQ1VSUkVOQ1kifSwgeyJDVVJSRU5DWSJ9KSwKICAgICAgICAoeyJTV0lGVF9DT0RFIn0sIHsiU1dJRlRfQ09ERSJ9KSwKICAgICAgICAoeyJVU19JVElOIn0sIHsiVVNfSVRJTiJ9KSwKICAgICAgICAoeyJVU19CQU5LX05VTUJFUiJ9LCB7IlVTX0JBTktfTlVNQkVSIn0pLAogICAgICAgICh7IlVTX0RSSVZFUl9MSUNFTlNFIn0sIHsiVVNfRFJJVkVSX0xJQ0VOU0UifSksCiAgICBdCgogICAgX0RFRkFVTFRfTU9ERUxfTEFOR1VBR0VTID0gewogICAgICAgICJlbiI6ICJiZWtpL2ZsYWlyLXBpaS1kaXN0aWxiZXJ0IiwKICAgIH0KCiAgICBfREVGQVVMVF9QUkVTSURJT19FUVVJVkFMRU5DRVMgPSB7CiAgICAgICAgIlBFUiI6ICJQRVJTT04iLAogICAgICAgICJMT0MiOiAiTE9DQVRJT04iLAogICAgICAgICJPUkciOiAiT1JHQU5JWkFUSU9OIiwKICAgICAgICAiTlJPUCI6ICJOUlAiLAogICAgICAgICJVUkwiOiAiVVJMIiwKICAgICAgICAiVVNfSVRJTiI6ICJVU19JVElOIiwKICAgICAgICAiVVNfUEFTU1BPUlQiOiAiVVNfUEFTU1BPUlQiLAogICAgICAgICJJQkFOX0NPREUiOiAiSUJBTl9DT0RFIiwKICAgICAgICAiSVBfQUREUkVTUyI6ICJJUF9BRERSRVNTIiwKICAgICAgICAiRU1BSUxfQUREUkVTUyI6ICJFTUFJTCIsCiAgICAgICAgIlVTX0RSSVZFUl9MSUNFTlNFIjogIlVTX0RSSVZFUl9MSUNFTlNFIiwKICAgICAgICAiVVNfQkFOS19OVU1CRVIiOiAiVVNfQkFOS19OVU1CRVIiLAogICAgfQoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgIHN1cHBvcnRlZF9sYW5ndWFnZTogc3RyID0gImVuIiwKICAgICAgICBzdXBwb3J0ZWRfZW50aXRpZXM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICAgICAgY2hlY2tfbGFiZWxfZ3JvdXBzOiBUdXBsZVtTZXQsIFNldF0gPSBOb25lLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSBGbGFpclJlY29nbml6ZXIuCgogICAgICAgIDpwYXJhbSBzdXBwb3J0ZWRfbGFuZ3VhZ2U6IExhbmd1YWdlIHRvIHVzZQogICAgICAgIDpwYXJhbSBzdXBwb3J0ZWRfZW50aXRpZXM6IEVudGl0aWVzIHRvIHVzZQogICAgICAgIDpwYXJhbSBjaGVja19sYWJlbF9ncm91cHM6IExhYmVsIGdyb3VwcyB0byBjaGVjawoKICAgICAgICA6cmV0dXJuczogRmxhaXJSZWNvZ25pemVyIG9iamVjdAoKICAgICAgICAiIiIKICAgICAgICBzZWxmLmNoZWNrX2xhYmVsX2dyb3VwcyA9IGNoZWNrX2xhYmVsX2dyb3VwcyBvciBzZWxmLl9ERUZBVUxUX0NIRUNLX0xBQkVMX0dST1VQUwoKICAgICAgICBzdXBwb3J0ZWRfZW50aXRpZXMgPSBzdXBwb3J0ZWRfZW50aXRpZXMgb3Igc2VsZi5SRUNPR05JWkFCTEVfRU5USVRJRVMKICAgICAgICBzZWxmLm1vZGVsID0gZmwubW9kZWxzLlNlcXVlbmNlVGFnZ2VyLmxvYWQoCiAgICAgICAgICAgIHNlbGYuX0RFRkFVTFRfTU9ERUxfTEFOR1VBR0VTLmdldChzdXBwb3J0ZWRfbGFuZ3VhZ2UpCiAgICAgICAgKQoKICAgICAgICBzdXBlcigpLl9faW5pdF9fKAogICAgICAgICAgICBzdXBwb3J0ZWRfZW50aXRpZXM9c3VwcG9ydGVkX2VudGl0aWVzLAogICAgICAgICAgICBzdXBwb3J0ZWRfbGFuZ3VhZ2U9c3VwcG9ydGVkX2xhbmd1YWdlLAogICAgICAgICAgICBuYW1lPSJGbGFpciBBbmFseXRpY3MiLAogICAgICAgICkKCiAgICAjIG1haW4gbWV0aG9kIGZvciB0aGUgcmVjb2duaXplcgogICAgZGVmIGFuYWx5emUoCiAgICAgICAgc2VsZiwKICAgICAgICB0ZXh0OiBzdHIsCiAgICAgICAgZW50aXRpZXM6IExpc3Rbc3RyXSwKICAgICAgICBubHBfYXJ0aWZhY3RzOiBwYS5ubHBfZW5naW5lLk5scEFydGlmYWN0cyA9IE5vbmUsCiAgICApIC0+IExpc3RbcGEuUmVjb2duaXplclJlc3VsdF06CiAgICAgICAgIiIiCiAgICAgICAgQW5hbHl6ZSB0ZXh0IGFuZCByZXR1cm4gdGhlIHJlc3VsdHMuCgogICAgICAgIDpwYXJhbSB0ZXh0OiAgICAgICAgICBUaGUgdGV4dCBmb3IgYW5hbHlzaXMuCiAgICAgICAgOnBhcmFtIGVudGl0aWVzOiAgICAgIFRoZSBsaXN0IG9mIGVudGl0aWVzIHRvIHJlY29nbml6ZS4KICAgICAgICA6cGFyYW0gbmxwX2FydGlmYWN0czogTm90IHVzZWQgYnkgdGhpcyByZWNvZ25pemVyIGJ1dCBuZWVkZWQgZm9yIHRoZSBpbnRlcmZhY2UuCiAgICAgICAgOnBhcmFtIGxhbmd1YWdlOiAgICAgIFRleHQgbGFuZ3VhZ2UuIFN1cHBvcnRlZCBsYW5ndWFnZXMgaW4gTU9ERUxfTEFOR1VBR0VTCgogICAgICAgIDpyZXR1cm5zOiBUaGUgbGlzdCBvZiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IGNvbnN0cnVjdGVkIGZyb20gdGhlIHJlY29nbml6ZWQgRmxhaXIgZGV0ZWN0aW9ucy4KICAgICAgICAiIiIKCiAgICAgICAgcmVzdWx0cyA9IFtdCgogICAgICAgIHNlbnRlbmNlcyA9IGZsLmRhdGEuU2VudGVuY2UodGV4dCkKICAgICAgICBzZWxmLm1vZGVsLnByZWRpY3Qoc2VudGVuY2VzKQoKICAgICAgICAjIElmIHRoZXJlIGFyZSBubyBzcGVjaWZpYyBsaXN0IG9mIGVudGl0aWVzLCB3ZSB3aWxsIGxvb2sgZm9yIGFsbCBvZiBpdC4KICAgICAgICBpZiBub3QgZW50aXRpZXM6CiAgICAgICAgICAgIGVudGl0aWVzID0gc2VsZi5zdXBwb3J0ZWRfZW50aXRpZXMKCiAgICAgICAgIyBHbyBvdmVyIHRoZSBlbnRpdGllcyBhbmQgY2hlY2sgaWYgdGhleSBhcmUgaW4gdGhlIHN1cHBvcnRlZCBlbnRpdGllcyBsaXN0LgogICAgICAgIGZvciBlbnRpdHkgaW4gZW50aXRpZXM6CiAgICAgICAgICAgIGlmIGVudGl0eSBub3QgaW4gc2VsZi5zdXBwb3J0ZWRfZW50aXRpZXM6CiAgICAgICAgICAgICAgICBjb250aW51ZQoKICAgICAgICAgICAgIyBHbyBvdmVyIHRoZSBzZW50ZW5jZXMgYW5kIGNoZWNrIGlmIHRoZSBlbnRpdHkgaXMgaW4gdGhlIHNlbnRlbmNlLgogICAgICAgICAgICBmb3IgZW50IGluIHNlbnRlbmNlcy5nZXRfc3BhbnMoIm5lciIpOgogICAgICAgICAgICAgICAgaWYgbm90IHNlbGYuX19jaGVja19sYWJlbCgKICAgICAgICAgICAgICAgICAgICBlbnRpdHksIGVudC5sYWJlbHNbMF0udmFsdWUsIHNlbGYuY2hlY2tfbGFiZWxfZ3JvdXBzCiAgICAgICAgICAgICAgICApOgogICAgICAgICAgICAgICAgICAgIGNvbnRpbnVlCgogICAgICAgICAgICAgICAgIyBJZiB0aGUgZW50aXR5IGlzIGluIHRoZSBzZW50ZW5jZSwgd2Ugd2lsbCBhZGQgaXQgdG8gdGhlIHJlc3VsdHMuCiAgICAgICAgICAgICAgICB0ZXh0dWFsX2V4cGxhbmF0aW9uID0gc2VsZi5fREVGQVVMVF9FWFBMQU5BVElPTi5mb3JtYXQoCiAgICAgICAgICAgICAgICAgICAgZW50LmxhYmVsc1swXS52YWx1ZQogICAgICAgICAgICAgICAgKQoKICAgICAgICAgICAgICAgICMgQnVpbGQgdGhlIGV4cGxhbmF0aW9uIGZvciB0aGUgcmVzdWx0CiAgICAgICAgICAgICAgICBleHBsYW5hdGlvbiA9IHNlbGYuX2J1aWxkX2ZsYWlyX2V4cGxhbmF0aW9uKAogICAgICAgICAgICAgICAgICAgIHJvdW5kKGVudC5zY29yZSwgMiksIHRleHR1YWxfZXhwbGFuYXRpb24KICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICBmbGFpcl9yZXN1bHQgPSBzZWxmLl9jb252ZXJ0X3RvX3JlY29nbml6ZXJfcmVzdWx0KGVudCwgZXhwbGFuYXRpb24pCgogICAgICAgICAgICAgICAgcmVzdWx0cy5hcHBlbmQoZmxhaXJfcmVzdWx0KQoKICAgICAgICByZXR1cm4gcmVzdWx0cwoKICAgIGRlZiBfY29udmVydF90b19yZWNvZ25pemVyX3Jlc3VsdCgKICAgICAgICBzZWxmLCBlbnRpdHk6IGZsLmRhdGEuU3BhbiwgZXhwbGFuYXRpb246IHN0cgogICAgKSAtPiBwYS5SZWNvZ25pemVyUmVzdWx0OgogICAgICAgICIiIgogICAgICAgIENvbnZlcnQgRmxhaXIgcmVzdWx0IHRvIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQuCgogICAgICAgIDpwYXJhbSBlbnRpdHk6ICAgICAgRmxhaXIgZW50aXR5IG9mIFNwYW4KICAgICAgICA6cGFyYW0gZXhwbGFuYXRpb246IFByZXNpZGlvIEFuYWx5c2lzRXhwbGFuYXRpb24KCiAgICAgICAgOnJldHVybnM6IFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQKICAgICAgICAiIiIKCiAgICAgICAgIyBDb252ZXJ0IHRoZSBlbnRpdHkgdHlwZSB0byBQcmVzaWRpbyBlbnRpdHkgdHlwZQogICAgICAgIGVudGl0eV90eXBlID0gc2VsZi5fREVGQVVMVF9QUkVTSURJT19FUVVJVkFMRU5DRVMuZ2V0KGVudGl0eS50YWcsIGVudGl0eS50YWcpCgogICAgICAgICMgQ29udmVydCB0aGUgc2NvcmUgdG8gUHJlc2lkaW8gc2NvcmUKICAgICAgICBmbGFpcl9zY29yZSA9IHJvdW5kKGVudGl0eS5zY29yZSwgMikKCiAgICAgICAgIyBDcmVhdGUgdGhlIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgZnJvbSB0aGUgRmxhaXIgZW50aXR5CiAgICAgICAgZmxhaXJfcmVzdWx0cyA9IHBhLlJlY29nbml6ZXJSZXN1bHQoCiAgICAgICAgICAgIGVudGl0eV90eXBlPWVudGl0eV90eXBlLAogICAgICAgICAgICBzdGFydD1lbnRpdHkuc3RhcnRfcG9zaXRpb24sCiAgICAgICAgICAgIGVuZD1lbnRpdHkuZW5kX3Bvc2l0aW9uLAogICAgICAgICAgICBzY29yZT1mbGFpcl9zY29yZSwKICAgICAgICAgICAgYW5hbHlzaXNfZXhwbGFuYXRpb249ZXhwbGFuYXRpb24sCiAgICAgICAgKQoKICAgICAgICByZXR1cm4gZmxhaXJfcmVzdWx0cwoKICAgIGRlZiBfYnVpbGRfZmxhaXJfZXhwbGFuYXRpb24oCiAgICAgICAgc2VsZiwgb3JpZ2luYWxfc2NvcmU6IGZsb2F0LCBleHBsYW5hdGlvbjogc3RyCiAgICApIC0+IHBhLkFuYWx5c2lzRXhwbGFuYXRpb246CiAgICAgICAgIiIiCiAgICAgICAgQ3JlYXRlIGV4cGxhbmF0aW9uIGZvciB3aHkgdGhpcyByZXN1bHQgd2FzIGRldGVjdGVkLgoKICAgICAgICA6cGFyYW0gb3JpZ2luYWxfc2NvcmU6IFNjb3JlIGdpdmVuIGJ5IHRoaXMgcmVjb2duaXplcgogICAgICAgIDpwYXJhbSBleHBsYW5hdGlvbjogICAgRXhwbGFuYXRpb24gc3RyaW5nCgogICAgICAgIDpyZXR1cm5zOiBQcmVzaWRpbyBBbmFseXNpc0V4cGxhbmF0aW9uCiAgICAgICAgIiIiCgogICAgICAgICMgQ3JlYXRlIHRoZSBQcmVzaWRpbyBBbmFseXNpc0V4cGxhbmF0aW9uIGZvciB0aGUgcmVzdWx0CiAgICAgICAgZXhwbGFuYXRpb24gPSBwYS5BbmFseXNpc0V4cGxhbmF0aW9uKAogICAgICAgICAgICByZWNvZ25pemVyPXNlbGYuX19jbGFzc19fLl9fbmFtZV9fLAogICAgICAgICAgICBvcmlnaW5hbF9zY29yZT1vcmlnaW5hbF9zY29yZSwKICAgICAgICAgICAgdGV4dHVhbF9leHBsYW5hdGlvbj1leHBsYW5hdGlvbiwKICAgICAgICApCiAgICAgICAgcmV0dXJuIGV4cGxhbmF0aW9uCgogICAgIyBzYW5pdHkgY2hlY2sgb2YgdGhlIGVudGl0eSBhbmQgbGFiZWwgYmVmb3JlIHJlY29nbml0aW9uCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgX19jaGVja19sYWJlbCgKICAgICAgICBlbnRpdHk6IHN0ciwgbGFiZWw6IHN0ciwgY2hlY2tfbGFiZWxfZ3JvdXBzOiBUdXBsZVtTZXQsIFNldF0KICAgICkgLT4gYm9vbDoKICAgICAgICByZXR1cm4gYW55KAogICAgICAgICAgICBlbnRpdHkgaW4gZWdycCBhbmQgbGFiZWwgaW4gbGdycCBmb3IgZWdycCwgbGdycCBpbiBjaGVja19sYWJlbF9ncm91cHMKICAgICAgICApCgoKIyBnZXQgdGhlIGFuYWx5emVyIGVuZ2luZSBiYXNlZCBvbiB0aGUgbW9kZWwKZGVmIF9nZXRfYW5hbHl6ZXJfZW5naW5lKAogICAgbW9kZWw6IHN0ciA9IE5vbmUsIGVudGl0aWVzOiBMaXN0W3N0cl0gPSBOb25lCikgLT4gcGEuQW5hbHl6ZXJFbmdpbmU6CiAgICAiIiIKICAgIFJldHVybiBwYS5BbmFseXplckVuZ2luZS4KCiAgICA6cGFyYW0gbW9kZWw6IFRoZSBtb2RlbCB0byB1c2UuIENhbiBiZSAic3BhY3kiLCAiZmxhaXIiLCAicGF0dGVybiIgb3IgIndob2xlIi4KICAgIDpwYXJhbSBlbnRpdGllczogVGhlIGxpc3Qgb2YgZW50aXRpZXMgdG8gdXNlLgoKICAgIDpyZXR1cm5zOiBwYS5BbmFseXplckVuZ2luZQogICAgIiIiCiAgICAjIHJlY29nbml6ZXIgcmVnaXN0cnkgdGhhdCBjYW4gc3RvcmUgbXVsdGlwbGUgcmVjb2duaXplcnMKICAgIHJlZ2lzdHJ5ID0gcGEuUmVjb2duaXplclJlZ2lzdHJ5KCkKICAgIGlmIG1vZGVsID09IE1vZGVscy5TUEFDWToKICAgICAgICAjIGN1c3RvbSBzcGFjeSByZWNvZ25pemVyCiAgICAgICAgc3BhY3lfcmVjb2duaXplciA9IEN1c3RvbVNwYWN5UmVjb2duaXplcigpCiAgICAgICAgIyBhZGQgdGhlIGN1c3RvbSBidWlsZCBzcGFjeSByZWNvZ25pemVyCiAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIoc3BhY3lfcmVjb2duaXplcikKICAgIGVsaWYgbW9kZWwgPT0gTW9kZWxzLkZMQUlSOgogICAgICAgICMgcHJlLXRyYWluZWQgZmxhaXIgcmVjb2duaXplcgogICAgICAgIGZsYWlyX3JlY29nbml6ZXIgPSBGbGFpclJlY29nbml6ZXIoKQogICAgICAgICMgYWRkIHRoZSBjdXN0b20gYnVpbGQgZmxhaXIgcmVjb2duaXplcgogICAgICAgIHJlZ2lzdHJ5LmFkZF9yZWNvZ25pemVyKGZsYWlyX3JlY29nbml6ZXIpCiAgICBlbGlmIG1vZGVsID09IE1vZGVscy5QQVRURVJOOgogICAgICAgICMgYWRkIHRoZSBwYXR0ZXJuIHJlY29nbml6ZXIKICAgICAgICBwYXR0ZXJuX3JlY29nbml6ZXJfZmFjdG9yeSA9IFBhdHRlcm5SZWNvZ25pemVyRmFjdG9yeSgpCiAgICAgICAgZm9yIHJlY29nbml6ZXIgaW4gcGF0dGVybl9yZWNvZ25pemVyX2ZhY3RvcnkuX2NyZWF0ZV9wYXR0ZXJuX3JlY29nbml6ZXIoKToKICAgICAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIocmVjb2duaXplcikKICAgIGVsaWYgbW9kZWwgPT0gTW9kZWxzLldIT0xFOgogICAgICAgIHNwYWN5X3JlY29nbml6ZXIgPSBDdXN0b21TcGFjeVJlY29nbml6ZXIoKQogICAgICAgIGZsYWlyX3JlY29nbml6ZXIgPSBGbGFpclJlY29nbml6ZXIoKQogICAgICAgIHJlZ2lzdHJ5LmFkZF9yZWNvZ25pemVyKHNwYWN5X3JlY29nbml6ZXIpCiAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIoZmxhaXJfcmVjb2duaXplcikKICAgICAgICAjIGFkZCB0aGUgcGF0dGVybiByZWNvZ25pemVyCiAgICAgICAgcGF0dGVybl9yZWNvZ25pemVyX2ZhY3RvcnkgPSBQYXR0ZXJuUmVjb2duaXplckZhY3RvcnkoKQogICAgICAgIGZvciByZWNvZ25pemVyIGluIHBhdHRlcm5fcmVjb2duaXplcl9mYWN0b3J5Ll9jcmVhdGVfcGF0dGVybl9yZWNvZ25pemVyKCk6CiAgICAgICAgICAgIHJlZ2lzdHJ5LmFkZF9yZWNvZ25pemVyKHJlY29nbml6ZXIpCiAgICBlbGlmIG5vdCBtb2RlbCBhbmQgZW50aXRpZXM6CiAgICAgICAgaWYgc2V0KGVudGl0aWVzKSAmIEN1c3RvbVNwYWN5UmVjb2duaXplci5SRUNPR05JWkFCTEVfRU5USVRJRVM6CiAgICAgICAgICAgIHNwYWN5X3JlY29nbml6ZXIgPSBDdXN0b21TcGFjeVJlY29nbml6ZXIoKQogICAgICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihzcGFjeV9yZWNvZ25pemVyKQogICAgICAgIGlmIHNldChlbnRpdGllcykgJiBGbGFpclJlY29nbml6ZXIuUkVDT0dOSVpBQkxFX0VOVElUSUVTOgogICAgICAgICAgICBmbGFpcl9yZWNvZ25pemVyID0gRmxhaXJSZWNvZ25pemVyKCkKICAgICAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIoZmxhaXJfcmVjb2duaXplcikKICAgICAgICAjIGFkZCB0aGUgcGF0dGVybiByZWNvZ25pemVyCiAgICAgICAgaWYgc2V0KGVudGl0aWVzKSAmIChzZXQoUGF0dGVyblJlY29nbml6ZXJGYWN0b3J5LlJFQ09HTklaQUJMRV9FTlRJVElFUy5rZXlzKCkpKToKICAgICAgICAgICAgcGF0dGVybl9yZWNvZ25pemVyX2ZhY3RvcnkgPSBQYXR0ZXJuUmVjb2duaXplckZhY3RvcnkoKQogICAgICAgICAgICBmb3IgcmVjb2duaXplciBpbiBwYXR0ZXJuX3JlY29nbml6ZXJfZmFjdG9yeS5fY3JlYXRlX3BhdHRlcm5fcmVjb2duaXplcigpOgogICAgICAgICAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIocmVjb2duaXplcikKICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJhcmd1bWVudCBvZiBtb2RlbCBhbmQgZW50aXRpZXMgY2FuIG5vdCBiZSBOb25lIGF0IHRoZSBzYW1lIHRpbWUiCiAgICAgICAgKQogICAgYW5hbHl6ZXIgPSBwYS5BbmFseXplckVuZ2luZSgKICAgICAgICByZWdpc3RyeT1yZWdpc3RyeSwKICAgICAgICBzdXBwb3J0ZWRfbGFuZ3VhZ2VzPVsiZW4iXSwKICAgICkKCiAgICBzdXBwb3J0ZWRfZW50aXRpZXMgPSBhbmFseXplci5nZXRfc3VwcG9ydGVkX2VudGl0aWVzKCkKCiAgICBpZiBlbnRpdGllcyBhbmQgbm90IGFsbChpdGVtIGluIHN1cHBvcnRlZF9lbnRpdGllcyBmb3IgaXRlbSBpbiBlbnRpdGllcyk6CiAgICAgICAgbm90X3N1cHBvcnRlZF9lbnRpdGllcyA9IFsKICAgICAgICAgICAgaXRlbSBmb3IgaXRlbSBpbiBlbnRpdGllcyBpZiBpdGVtIG5vdCBpbiBzdXBwb3J0ZWRfZW50aXRpZXMKICAgICAgICBdCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJUaGUgY3VycmVudCBtb2RlbCB7bW9kZWx9IGRvZXNuJ3Qgc3VwcG9ydCB0aGUgZm9sbG93aW5nIGVudGl0aWVzOiB7bm90X3N1cHBvcnRlZF9lbnRpdGllc30uICIKICAgICAgICAgICAgZiJTdXBwb3J0ZWQgZW50aXRpZXMgYXJlOiB7c3VwcG9ydGVkX2VudGl0aWVzfSIKICAgICAgICApCiAgICByZXR1cm4gYW5hbHl6ZXIKCgpkZWYgX2dldF9hbm9ueW1pemVyX2VuZ2luZSgpIC0+IHByZV9hbm95bWl6ZXIuQW5vbnltaXplckVuZ2luZToKICAgICIiIgogICAgUmV0dXJuIEFub255bWl6ZXJFbmdpbmUuCgogICAgOnJldHVybnM6IFRoZSBBbm9ueW1pemVyRW5naW5lLgogICAgIiIiCiAgICByZXR1cm4gcHJlX2Fub3ltaXplci5Bbm9ueW1pemVyRW5naW5lKCkKCgpkZWYgX2Fub255bWl6ZSgKICAgIHRleHQ6IHN0ciwKICAgIGFuYWx5emVfcmVzdWx0czogTGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XSwKICAgIGVudGl0eV9vcGVyYXRvcl9tYXA6IGRpY3QgPSBOb25lLAogICAgaXNfZnVsbF90ZXh0OiBib29sID0gVHJ1ZSwKKSAtPiBzdHI6CiAgICAiIiIKICAgIEFub255bWl6ZSBpZGVudGlmaWVkIGlucHV0IHVzaW5nIFByZXNpZGlvIEFib255bWl6ZXIuCgogICAgOnBhcmFtIHRleHQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgIDpwYXJhbSBhbmFseXplX3Jlc3VsdHM6ICAgICBUaGUgbGlzdCBvZiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IGNvbnN0cnVjdGVkIGZyb20KICAgIDpwYXJhbSBlbnRpdHlfb3BlcmF0b3JfbWFwOiBUaGUgZW50aXR5X29wZXJhdG9yX21hcCBpcyBhIGRpY3Rpb25hcnkgdGhhdCBtYXBzIGVudGl0eSB0byBvcGVyYXRvciBuYW1lIGFuZCBvcGVyYXRvciBwYXJhbXMuCiAgICA6cGFyYW0gaXNfZnVsbF90ZXh0OiAgICAgICAgV2hldGhlciB0aGUgdGV4dCBpcyBmdWxsIHRleHQgb3Igbm90LgoKICAgIDpyZXR1cm5zOiBUaGUgYW5vbnltaXplZCB0ZXh0LgogICAgIiIiCiAgICBpZiBub3QgdGV4dDoKICAgICAgICByZXR1cm4gIiIKCiAgICBhbm9ueW1pemVyX2VuZ2luZSA9IF9nZXRfYW5vbnltaXplcl9lbmdpbmUoKQogICAgaWYgbm90IGVudGl0eV9vcGVyYXRvcl9tYXA6CiAgICAgICAgb3BlcmF0b3JzID0gTm9uZQogICAgZWxzZToKICAgICAgICAjIENyZWF0ZSBPcGVyYXRvckNvbmZpZyBiYXNlZCBvbiB0aGUgZW50aXR5X29wZXJhdG9yX21hcAogICAgICAgIG9wZXJhdG9ycyA9IHsKICAgICAgICAgICAgZW50aXR5OiBPcGVyYXRvckNvbmZpZyhvcGVyYXRvcl9uYW1lLCBvcGVyYXRvcl9wYXJhbXMpCiAgICAgICAgICAgIGZvciBlbnRpdHksIChvcGVyYXRvcl9uYW1lLCBvcGVyYXRvcl9wYXJhbXMpIGluIGVudGl0eV9vcGVyYXRvcl9tYXAuaXRlbXMoKQogICAgICAgIH0KCiAgICBpZiBpc19mdWxsX3RleHQ6CiAgICAgICAgIyBBbm9ueW1pemUgdGhlIGVudGlyZSB0ZXh0CiAgICAgICAgcmV0dXJuIGFub255bWl6ZXJfZW5naW5lLmFub255bWl6ZSgKICAgICAgICAgICAgdGV4dD10ZXh0LCBhbmFseXplcl9yZXN1bHRzPWFuYWx5emVfcmVzdWx0cywgb3BlcmF0b3JzPW9wZXJhdG9ycwogICAgICAgICkudGV4dAogICAgIyBUb2tlbml6ZSB0aGUgdGV4dCB0byBzZW50ZW5jZXMKICAgIHNlbnRlbmNlcyA9IG5sdGsuc2VudF90b2tlbml6ZSh0ZXh0KQogICAgYW5vbnltaXplZF9zZW50ZW5jZXMgPSBbXQogICAgY3VycmVudF9pZHggPSAwCgogICAgIyBGaW5kIHRoZSBzZW50ZW5jZSB0aGF0IGhhcyBwaWkgZW50aXR5CiAgICBmb3Igc2VudGVuY2UgaW4gc2VudGVuY2VzOgogICAgICAgIHN0YXJ0X2lkeCA9IGN1cnJlbnRfaWR4CiAgICAgICAgZW5kX2lkeCA9IHN0YXJ0X2lkeCArIGxlbihzZW50ZW5jZSkKCiAgICAgICAgIyBHZXQgdGhlIGVudGl0aWVzIHRoYXQgYXJlIGluIHRoZSBzZW50ZW5jZSwgdXBkYXRlIGh0ZSBzdGFydF9pZHggYW5kIGVuZF9pZHgKICAgICAgICBzZW50ZW5jZV9yZXN1bHRzID0gWwogICAgICAgICAgICBwYS5SZWNvZ25pemVyUmVzdWx0KAogICAgICAgICAgICAgICAgcmVzdWx0LmVudGl0eV90eXBlLAogICAgICAgICAgICAgICAgc3RhcnQ9cmVzdWx0LnN0YXJ0IC0gc3RhcnRfaWR4LAogICAgICAgICAgICAgICAgZW5kPXJlc3VsdC5lbmQgLSBzdGFydF9pZHgsCiAgICAgICAgICAgICAgICBzY29yZT1yZXN1bHQuc2NvcmUsCiAgICAgICAgICAgICkKICAgICAgICAgICAgZm9yIHJlc3VsdCBpbiBhbmFseXplX3Jlc3VsdHMKICAgICAgICAgICAgaWYgcmVzdWx0LnN0YXJ0ID49IHN0YXJ0X2lkeCBhbmQgcmVzdWx0LmVuZCA8PSBlbmRfaWR4CiAgICAgICAgXQoKICAgICAgICAjIElmIFBJSSBpcyBkZXRlY3RlZAogICAgICAgIGlmIHNlbnRlbmNlX3Jlc3VsdHM6CiAgICAgICAgICAgIGFub255bWl6ZWRfc2VudGVuY2UgPSBhbm9ueW1pemVyX2VuZ2luZS5hbm9ueW1pemUoCiAgICAgICAgICAgICAgICB0ZXh0PXNlbnRlbmNlLCBhbmFseXplcl9yZXN1bHRzPXNlbnRlbmNlX3Jlc3VsdHMsIG9wZXJhdG9ycz1vcGVyYXRvcnMKICAgICAgICAgICAgKS50ZXh0CiAgICAgICAgICAgIGFub255bWl6ZWRfc2VudGVuY2VzLmFwcGVuZChhbm9ueW1pemVkX3NlbnRlbmNlKQoKICAgICAgICBjdXJyZW50X2lkeCA9IGVuZF9pZHgKCiAgICByZXR1cm4gIiAiLmpvaW4oYW5vbnltaXplZF9zZW50ZW5jZXMpCgoKZGVmIF9nZXRfdG9rZW5zKAogICAgdGV4dDogc3RyLCBhbmFseXplX3Jlc3VsdHM6IExpc3RbcGEuUmVjb2duaXplclJlc3VsdF0sIGlzX2Z1bGw6IGJvb2wgPSBUcnVlCikgLT4gTGlzdFtzdHJdOgogICAgIiIiCiAgICBHZXQgdGhlIGZ1bGwgdG9rZW5zIG9yIG9ubHkgY29udGFpbnMgdGhlIGVudGl0aWVzIHRoYXQgY2FuIGZvcm0gYSBzZW50ZW5jZS4KCiAgICA6cGFyYW0gdGV4dDogICAgICAgICAgICBUaGUgdGV4dCBmb3IgYW5hbHlzaXMuCiAgICA6cGFyYW0gYW5hbHl6ZV9yZXN1bHRzOiBUaGUgbGlzdCBvZiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IGNvbnN0cnVjdGVkIGZyb20KICAgIDpwYXJhbSBpc19mdWxsOiAgICAgICAgIFdoZXRoZXIgcmV0dXJuIGZ1bGwgdG9rZW5zIG9yIGp1c3QgdGhlIHRva2VucyB0aGF0IG9ubHkgY29udGFpbnMgdGhlIGVudGl0aWVzIHRoYXQgY2FuIGZvcm0gYSBzZW50ZW5jZS4KCiAgICA6cmV0dXJuczogVGhlIHRva2Vucy4KICAgICIiIgoKICAgIHRva2VucyA9IFtdCiAgICAjIHNvcnQgYnkgc3RhcnQgaW5kZXgKICAgIHJlc3VsdHMgPSBzb3J0ZWQoYW5hbHl6ZV9yZXN1bHRzLCBrZXk9bGFtYmRhIHg6IHguc3RhcnQpCiAgICBmb3IgaSwgcmVzIGluIGVudW1lcmF0ZShyZXN1bHRzKToKICAgICAgICBpZiBpID09IDA6CiAgICAgICAgICAgIHRva2Vucy5hcHBlbmQodGV4dFs6IHJlcy5zdGFydF0pCgogICAgICAgICMgYXBwZW5kIGVudGl0eSB0ZXh0IGFuZCBlbnRpdHkgdHlwZQogICAgICAgIHRva2Vucy5hcHBlbmQoKHRleHRbcmVzLnN0YXJ0IDogcmVzLmVuZF0sIHJlcy5lbnRpdHlfdHlwZSkpCgogICAgICAgICMgaWYgYW5vdGhlciBlbnRpdHkgY29taW5nIGkuZS4gd2UncmUgbm90IGF0IHRoZSBsYXN0IHJlc3VsdHMgZWxlbWVudCwKICAgICAgICAjIGFkZCB0ZXh0IHVwIHRvIG5leHQgZW50aXR5CiAgICAgICAgaWYgaSAhPSBsZW4ocmVzdWx0cykgLSAxOgogICAgICAgICAgICB0b2tlbnMuYXBwZW5kKHRleHRbcmVzLmVuZCA6IHJlc3VsdHNbaSArIDFdLnN0YXJ0XSkKICAgICAgICAjIGlmIG5vIG1vcmUgZW50aXRpZXMgY29taW5nLCBhZGQgYWxsIHJlbWFpbmluZyB0ZXh0CiAgICAgICAgZWxzZToKICAgICAgICAgICAgdG9rZW5zLmFwcGVuZCh0ZXh0W3Jlcy5lbmQgOl0pCgogICAgIyBnZXQgdGhlIHRva2VucyB0aGF0IG9ubHkgY29udGFpbnMgdGhlIGVudGl0aWVzIHRoYXQgY2FuIGZvcm0gYSBzZW50ZW5jZQogICAgcGFydF9hbm5vbnRhdGVkX3Rva2VucyA9IFtdCiAgICBpZiBub3QgaXNfZnVsbDoKICAgICAgICBsYXN0X2VuZF9zZW50ZW5jZSA9IDAKICAgICAgICBmb3IgaSwgdG9rZW4gaW4gZW51bWVyYXRlKHRva2Vucyk6CiAgICAgICAgICAgIGlmIGFueShpdGVtIGluIHRva2VuIGZvciBpdGVtIGluIFsiLiIsICIhIiwgIj8iXSkgYW5kIGFueSgKICAgICAgICAgICAgICAgIHR5cGUoaXRlbSkgaXMgdHVwbGUgZm9yIGl0ZW0gaW4gdG9rZW5zW2xhc3RfZW5kX3NlbnRlbmNlOmldCiAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICBwYXJ0X2Fubm9udGF0ZWRfdG9rZW5zLmFwcGVuZCh0b2tlbnNbbGFzdF9lbmRfc2VudGVuY2U6aV0pCiAgICAgICAgICAgICAgICBsYXN0X2VuZF9zZW50ZW5jZSA9IGkKICAgICAgICByZXR1cm4gcGFydF9hbm5vbnRhdGVkX3Rva2VucwogICAgcmV0dXJuIHRva2VucwoKCmRlZiBfYW5ub3RhdGUoCiAgICB0ZXh0OiBzdHIsIHN0X2FuYWx5emVfcmVzdWx0czogTGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XSwgaXNfZnVsbF9odG1sOiBib29sID0gVHJ1ZQopIC0+IExpc3Rbc3RyXToKICAgICIiIgogICAgQW5ub3RhdGUgaWRlbnRpZmllZCBpbnB1dCB1c2luZyBQcmVzaWRpbyBBbm9ueW1pemVyLgoKICAgIDpwYXJhbSB0ZXh0OiAgICAgICAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgIDpwYXJhbSBzdF9hbmFseXplX3Jlc3VsdHM6IFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX2h0bWw6ICAgICAgIFdoZXRoZXIgZ2VuZXJhdGUgZnVsbCBodG1sIG9yIG5vdC4KCiAgICA6cmV0dXJuczogVGhlIGxpc3Qgb2YgdG9rZW5zIHdpdGggdGhlIGlkZW50aWZpZWQgZW50aXRpZXMuCgogICAgIiIiCiAgICByZXR1cm4gX2dldF90b2tlbnModGV4dCwgc3RfYW5hbHl6ZV9yZXN1bHRzLCBpc19mdWxsX2h0bWwpCgoKZGVmIF9wcm9jZXNzKAogICAgdGV4dDogc3RyLAogICAgbW9kZWw6IHBhLkFuYWx5emVyRW5naW5lLAogICAgc2NvcmVfdGhyZXNob2xkOiBmbG9hdCwKICAgIGVudGl0aWVzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgZW50aXRpZXNfb3BlcmF0b3JfbWFwOiBkaWN0ID0gTm9uZSwKICAgIGlzX2Z1bGxfdGV4dDogYm9vbCA9IFRydWUsCikgLT4gVHVwbGVbc3RyLCBzdHIsIHN0cl06CiAgICAiIiIKICAgIFByb2Nlc3MgdGhlIHRleHQgb2Ygc3RyIHVzaW5nIHRoZSBtb2RlbC4KCiAgICA6cGFyYW0gdHh0OiAgICAgICAgICAgICAgICAgICBUZXh0IHRvIHByb2Nlc3MKICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgIE1vZGVsIHRvIHVzZSBmb3IgcHJvY2Vzc2luZwogICAgOnBhcmFtIGVudGl0aWVzOiAgICAgICAgICAgICAgRW50aXRpZXMgdG8gcmVjb2duaXplCiAgICA6cGFyYW0gZW50aXRpZXNfb3BlcmF0b3JfbWFwOiBUaGUgZW50aXR5X29wZXJhdG9yX21hcCBpcyBhIGRpY3Rpb25hcnkgdGhhdCBtYXBzIGVudGl0eSB0byBvcGVyYXRvciBuYW1lIGFuZCBvcGVyYXRvciBwYXJhbXMuCiAgICA6cGFyYW0gc2NvcmVfdGhyZXNob2xkOiAgICAgICBUaGUgc2NvcmUgdGhyZXNob2xkIHRvIHVzZSBmb3IgcmVjb2duaXRpb24KICAgIDpwYXJhbSBpc19mdWxsX3RleHQ6ICAgICAgICAgIFdoZXRoZXIgdG8gcmV0dXJuIHRoZSBmdWxsIHRleHQgb3IganVzdCB0aGUgYW5ub3RhdGVkIHRleHQKCiAgICA6cmV0dXJuczogQSB0dXBsZSBvZjoKCiAgICAgICAgICAgICAgKiB0aGUgYW5vbnltaXplZCB0ZXh0CiAgICAgICAgICAgICAgKiB0aGUgbGlzdCBvZiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IGNvbnN0cnVjdGVkIGZyb20gYW5hbHlzaXMKICAgICIiIgoKICAgICMgZ2V0IHRoZSBhbmFseXplciBlbmdpbmUKCiAgICBhbmFseXplciA9IG1vZGVsCgogICAgIyBhbmFseXplIHRoZSB0ZXh0IHRoYXQgY2FuIGJlIHVzZWQgZm9yIGFub255bWl6YXRpb24KICAgIHJlc3VsdHMgPSBhbmFseXplci5hbmFseXplKAogICAgICAgIHRleHQ9dGV4dCwKICAgICAgICBsYW5ndWFnZT0iZW4iLAogICAgICAgIGVudGl0aWVzPWVudGl0aWVzLAogICAgICAgIHNjb3JlX3RocmVzaG9sZD1zY29yZV90aHJlc2hvbGQsCiAgICAgICAgcmV0dXJuX2RlY2lzaW9uX3Byb2Nlc3M9VHJ1ZSwKICAgICkKCiAgICAjIGFub255bWl6ZSB0aGUgdGV4dCwgcmVwbGFjZSB0aGUgcGlpIGVudGl0aWVzIHdpdGggdGhlIGxhYmVscwogICAgYW5vbnltaXplZF90ZXh0ID0gX2Fub255bWl6ZSh0ZXh0LCByZXN1bHRzLCBlbnRpdGllc19vcGVyYXRvcl9tYXAsIGlzX2Z1bGxfdGV4dCkKCiAgICByZXR1cm4gYW5vbnltaXplZF90ZXh0LCByZXN1bHRzCgoKZGVmIF9nZXRfc2luZ2xlX2h0bWwoCiAgICB0ZXh0OiBzdHIsIHJlc3VsdHM6IExpc3RbcGEuUmVjb2duaXplclJlc3VsdF0sIGlzX2Z1bGxfaHRtbDogYm9vbCA9IFRydWUKKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIGh0bWwgZm9yIGEgc2luZ2xlIHR4dCBmaWxlLgoKICAgIDpwYXJhbSB0ZXh0OiAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgIDpwYXJhbSByZXN1bHRzOiAgICAgIFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX2h0bWw6IFdoZXRoZXIgZ2VuZXJhdGUgZnVsbCBodG1sIG9yIG5vdC4KCiAgICA6cmV0dXJuczogVGhlIGh0bWwgc3RyaW5nIGZvciBhIHNpbmdsZSB0eHQgZmlsZS4KICAgICIiIgogICAgIyBjb252ZXJ0IHRoZSByZXN1bHRzIHRvIHRva2VucyB0byBnZW5lcmF0ZSB0aGUgaHRtbAogICAgdG9rZW5zID0gX2Fubm90YXRlKHRleHQsIHJlc3VsdHMsIGlzX2Z1bGxfaHRtbCkKICAgIGh0bWwgPSBhdF91dGlsLmdldF9hbm5vdGF0ZWRfaHRtbCgqdG9rZW5zKQoKICAgICMgYXZvaWQgdGhlIGVycm9yIGR1cmluZyByZW5kZXJpbmcgb2YgdGhlIFxuIGluIHRoZSBodG1sCiAgICBiYWNrc2xhc2hfY2hhciA9ICJcXCIKCiAgICBodG1sX3N0ciA9IGYiPHA+e2h0bWwucmVwbGFjZSgne2JhY2tzbGFzaF9jaGFyfW4nLCAnPGJyPicpfTwvcD4iCgogICAgcmV0dXJuIGh0bWxfc3RyCgoKZGVmIF9nZXRfc2luZ2xlX2pzb24ocmVzdWx0czogTGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XSwgaXNfZnVsbF9yZXBvcnQ6IGJvb2wgPSBUcnVlKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIGpzb24gZm9yIGEgc2luZ2xlIHR4dCBmaWxlLgoKICAgIDpwYXJhbSByZXN1bHRzOiAgICAgICAgVGhlIGxpc3Qgb2YgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBjb25zdHJ1Y3RlZCBmcm9tIGFuYWx5c2lzLgogICAgOnBhcmFtIGlzX2Z1bGxfcmVwb3J0OiBXaGV0aGVyIGdlbmVyYXRlIGZ1bGwganNvbiBvciBub3QuCgogICAgOnJldHVybnM6IFRoZSBqc29uIHN0cmluZyBmb3IgYSBzaW5nbGUgdHh0IGZpbGUuCiAgICAiIiIKICAgICMgZ2VuZXJhdGUgdGhlIHN0YXRzIHJlcG9ydCBpZiBuZWVkZWQKICAgIGlmIG5vdCBpc19mdWxsX3JlcG9ydDoKICAgICAgICBzdGF0cyA9IFtdCiAgICAgICAgIyBhZGQgdGhlIHNpbXBsaWZ5IHN0YXRzIGxvZ2ljIGhlcmUKICAgICAgICBmb3IgaXRlbSBpbiByZXN1bHRzOgogICAgICAgICAgICBpdGVtLmFuYWx5c2lzX2V4cGxhbmF0aW9uID0gTm9uZQogICAgICAgICAgICBzdGF0cy5hcHBlbmQoaXRlbSkKICAgIGVsc2U6CiAgICAgICAgc3RhdHMgPSByZXN1bHRzCgogICAgcmV0dXJuIHN0YXRzCgoKZGVmIF9nZXRfYWxsX2h0bWwoCiAgICB0eHRfY29udGVudDogZGljdCwKICAgIHJlc19kaWN0OiBkaWN0LAogICAgaXNfZnVsbF9odG1sOiBib29sID0gVHJ1ZSwKKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIGh0bWwgZm9yIGFsbCB0eHQgZmlsZXMuCgogICAgOnBhcmFtIHR4dF9jb250ZW50OiAgVGhlIGRpY3Rpb25hcnkgb2YgdHh0IGZpbGUgbmFtZSBhbmQgY29udGVudC4KICAgIDpwYXJhbSByZXNfZGljdDogICAgIFRoZSBkaWN0aW9uYXJ5IG9mIHR4dCBmaWxlIG5hbWUgYW5kIHRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX2h0bWw6IFdoZXRoZXIgZ2VuZXJhdGUgZnVsbCBodG1sIG9yIG5vdC4KCiAgICA6cmV0dXJuczogVGhlIGh0bWwgc3RyaW5nIGZvciBhbGwgdHh0IGZpbGVzLgoKICAgICIiIgogICAgIyBUaGVzZSBhcmUgcGxhY2Vob2xkZXIgZm9yIHRoZSBodG1sIHN0cmluZwogICAgaHRtbF9pbmRleCA9ICI8aHRtbD48aGVhZD48dGl0bGU+SGlnaGxpZ2h0ZWQgUGlpIEVudGl0aWVzPC90aXRsZT48L2hlYWQ+PGJvZHk+PGgxPkhpZ2hsaWdodGVkIFBpaSBFbnRpdGllczwvaDE+PHVsPiIKICAgIGh0bWxfY29udGVudCA9ICIiCiAgICBmb3IgdHh0X2ZpbGUsIHJlc3VsdHMgaW4gcmVzX2RpY3QuaXRlbXMoKToKICAgICAgICB0eHQgPSB0eHRfY29udGVudFt0eHRfZmlsZV0KICAgICAgICBodG1sX2luZGV4ICs9IGYiPGxpPjxhIGhyZWY9JyN7dHh0X2ZpbGV9Jz57dHh0X2ZpbGV9PC9hPjwvbGk+IgogICAgICAgIGh0bWxfY29udGVudCArPSBmIjxsaT48aDI+e3R4dF9maWxlfTwvaDI+PHA+e19nZXRfc2luZ2xlX2h0bWwodHh0LCByZXN1bHRzLCBpc19mdWxsX2h0bWwpfTwvcD48L2xpPiIKICAgIGh0bWxfaW5kZXggKz0gIjwvdWw+IgogICAgaHRtbF9yZXMgPSBmIntodG1sX2luZGV4fXtodG1sX2NvbnRlbnR9PC9ib2R5PjwvaHRtbD4iCgogICAgcmV0dXJuIGh0bWxfcmVzCgoKZGVmIF9nZXRfYWxsX3JwdChyZXNfZGljdDogZGljdCwgaXNfZnVsbF9yZXBvcnQ6IGJvb2wgPSBUcnVlKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIHN0YXRzIHJlcG9ydCBmb3IgYWxsIHR4dCBmaWxlcy4KCiAgICA6cGFyYW0gcmVzX2RpY3Q6ICAgICAgIFRoZSBkaWN0aW9uYXJ5IG9mIHR4dCBmaWxlIG5hbWUgYW5kIHRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX3JlcG9ydDogV2hldGhlciBnZW5lcmF0ZSBmdWxsIHJlcG9ydCBvciBub3QuCgogICAgOnJldHVybnM6IFRoZSBzdGF0cyByZXBvcnQgZm9yIGFsbCB0eHQgZmlsZXMuCiAgICAiIiIKICAgICMgVGhlc2UgYXJlIHBsYWNlaG9sZGVyIGZvciB0aGUganNvbiByZXBvcnQKICAgIHN0YXRzX2RpY3QgPSB7fQogICAgZm9yIHR4dF9maWxlLCByZXN1bHRzIGluIHJlc19kaWN0Lml0ZW1zKCk6CiAgICAgICAgbmV3X3N0YXRzID0gW10KICAgICAgICBmb3IgaXRlbSBpbiBfZ2V0X3NpbmdsZV9qc29uKHJlc3VsdHMsIGlzX2Z1bGxfcmVwb3J0KToKICAgICAgICAgICAgaWYgaXNfZnVsbF9yZXBvcnQ6CiAgICAgICAgICAgICAgICBpdGVtLmFuYWx5c2lzX2V4cGxhbmF0aW9uID0gaXRlbS5hbmFseXNpc19leHBsYW5hdGlvbi50b19kaWN0KCkKICAgICAgICAgICAgICAgIG5ld19zdGF0cy5hcHBlbmQoaXRlbS50b19kaWN0KCkpCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICB0bXBfZGljdCA9IGl0ZW0udG9fZGljdCgpCiAgICAgICAgICAgICAgICB0bXBfZGljdC5wb3AoImFuYWx5c2lzX2V4cGxhbmF0aW9uIikKICAgICAgICAgICAgICAgIHRtcF9kaWN0LnBvcCgicmVjb2duaXRpb25fbWV0YWRhdGEiKQogICAgICAgICAgICAgICAgbmV3X3N0YXRzLmFwcGVuZCh0bXBfZGljdCkKICAgICAgICBzdGF0c19kaWN0W3R4dF9maWxlXSA9IG5ld19zdGF0cwogICAgcmV0dXJuIHN0YXRzX2RpY3QKCgpkZWYgcmVjb2duaXplX3BpaSgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgaW5wdXRfcGF0aDogVW5pb25bc3RyLHBhdGhsaWIuUGF0aF0sCiAgICBvdXRwdXRfcGF0aDogc3RyLAogICAgb3V0cHV0X3N1ZmZpeDogc3RyLAogICAgaHRtbF9rZXk6IHN0ciwKICAgIHNjb3JlX3RocmVzaG9sZDogZmxvYXQsCiAgICBlbnRpdGllczogTGlzdFsKICAgICAgICBzdHIKICAgIF0gPSBOb25lLCAgIyBMaXN0IG9mIGVudGl0aWVzIHRvIHJlY29nbml6ZSwgZGVmYXVsdCBpcyByZWNvZ25pemluZyBhbGwKICAgIGVudGl0eV9vcGVyYXRvcl9tYXA6IGRpY3QgPSBOb25lLAogICAgbW9kZWw6IHN0ciA9IE5vbmUsCiAgICBnZW5lcmF0ZV9qc29uOiBib29sID0gVHJ1ZSwKICAgIGdlbmVyYXRlX2h0bWw6IGJvb2wgPSBUcnVlLAogICAgaXNfZnVsbF90ZXh0OiBib29sID0gVHJ1ZSwKICAgIGlzX2Z1bGxfaHRtbDogYm9vbCA9IFRydWUsCiAgICBpc19mdWxsX3JlcG9ydDogYm9vbCA9IFRydWUsCikgLT4gVHVwbGVbcGF0aGxpYi5QYXRoLCBkaWN0LCBkaWN0XToKICAgICIiIgogICAgV2FsayB0aHJvdWdoIHRoZSBpbnB1dCBwYXRoLCByZWNvZ25pemUgUElJIGluIHRleHQgYW5kIHN0b3JlIHRoZSBhbm9ueW1pemVkIHRleHQgaW4gdGhlIG91dHB1dCBwYXRoLiBHZW5lcmF0ZSB0aGUgaHRtbCB3aXRoIGRpZmZlcmVudCBjb2xvcnMgZm9yIGVhY2ggZW50aXR5LCBqc29uIHJlcG9ydCBvZiB0aGUgZXhwbGFpbmF0aW9uLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgVGhlIE1MUnVuIGNvbnRleHQuIHRoaXMgaXMgbmVlZGVkIGZvciBsb2cgdGhlIGFydGlmYWN0cy4KICAgIDpwYXJhbSBpbnB1dF9wYXRoOiAgICAgICAgICAgVGhlIGlucHV0IHBhdGggb2YgdGhlIHRleHQgZmlsZXMgbmVlZHMgdG8gYmUgYW5hbHl6aWVkLgogICAgOnBhcmFtIG91dHB1dF9wYXRoOiAgICAgICAgICBUaGUgb3V0cHV0IHBhdGggdG8gc3RvcmUgdGhlIGFub255bWl6ZWQgdGV4dC4KICAgIDpwYXJhbSBvdXRwdXRfc3VmZml4OiAgICAgICAgVGhlIHN1cmZpeCBvZiBvdXRwdXQga2V5IGZvciB0aGUgYW5vbnltaXplZCB0ZXh0LiBmb3IgZXhhbXBsZSBpZiB0aGUgaW5wdXQgZmlsZSBpcyBwaWkudHh0LCB0aGUgb3V0cHV0IGtleSBpcyBhbm95bWl6ZWQsIHRoZSBvdXRwdXQgZmlsZSBuYW1lIHdpbGwgYmUgcGlpX2Fub255bWl6ZWQudHh0LgogICAgOnBhcmFtIGh0bWxfa2V5OiAgICAgICAgICAgICBUaGUgaHRtbCBrZXkgZm9yIHRoZSBhcnRpZmFjdC4KICAgIDpwYXJhbSBzY29yZV90aHJlc2hvbGQ6ICAgICAgVGhlIHNjb3JlIHRocmVzaG9sZCB0byBtYXJrIHRoZSByZWNvZ25pdGlvbiBhcyB0cnVzdGVkLgogICAgOnBhcmFtIGVudGl0aWVzOiAgICAgICAgICAgICBUaGUgbGlzdCBvZiBlbnRpdGllcyB0byByZWNvZ25pemUuCiAgICA6cGFyYW0gZW50aXR5X29wZXJhdG9yX21hcDogIFRoZSBtYXAgb2YgZW50aXR5IHRvIG9wZXJhdG9yIChtYXNrLCByZWRhY3QsIHJlcGxhY2UsIGtlZXAsIGhhc2gsIGFuZCBpdHMgcGFyYW1zKQogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICBUaGUgbW9kZWwgdG8gdXNlLiBDYW4gYmUgInNwYWN5IiwgImZsYWlyIiwgInBhdHRlcm4iIG9yICJ3aG9sZSIuCiAgICA6cGFyYW0gZ2VuZXJhdGVfanNvbjogICAgICAgIFdoZXRoZXIgdG8gZ2VuZXJhdGUgdGhlIGpzb24gcmVwb3J0IG9mIHRoZSBleHBsYWluYXRpb24uCiAgICA6cGFyYW0gZ2VuZXJhdGVfaHRtbDogICAgICAgIFdoZXRoZXIgdG8gZ2VuZXJhdGUgdGhlIGh0bWwgcmVwb3J0IG9mIHRoZSBleHBsYWluYXRpb24uCiAgICA6cGFyYW0gaXNfZnVsbF90ZXh0OiAgICAgICAgIFdoZXRoZXIgdG8gcmV0dXJuIHRoZSBmdWxsIHRleHQgb3Igb25seSB0aGUgbWFza2VkIHRleHQuCiAgICA6cGFyYW0gaXNfZnVsbF9odG1sOiAgICAgICAgIFdoZXRoZXIgdG8gcmV0dXJuIHRoZSBmdWxsIGh0bWwgb3IganVzdCB0aGUgYW5ub3RhdGVkIHRleHQKICAgIDpwYXJhbSBpc19mdWxsX3JlcG9ydDogICAgICAgV2hldGhlciB0byByZXR1cm4gdGhlIGZ1bGwgcmVwb3J0IG9yIGp1c3QgdGhlIHNjb3JlIGFuZCBzdGFydCwgZW5kIGluZGV4CgogICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CgogICAgICAgICAgICAgICogUGF0aCB0byB0aGUgb3V0cHV0IGRpcmVjdG9yeQogICAgICAgICAgICAgICogVGhlIGpzb24gcmVwb3J0IG9mIHRoZSBleHBsYWluYXRpb24gKGlmIGdlbmVyYXRlX2pzb24gaXMgVHJ1ZSkKICAgICAgICAgICAgICAqIEEgZGljdGlvbmFyeSBvZiBlcnJvcnMgZmlsZXMgdGhhdCB3ZXJlIG5vdCBwcm9jZXNzZWQKCiAgICAiIiIKCiAgICAjIFNldCBvdXRwdXQgZGlyZWN0b3J5CiAgICBpZiBvdXRwdXRfcGF0aCBpcyBOb25lOgogICAgICAgIG91dHB1dF9wYXRoID0gdGVtcGZpbGUubWtkdGVtcCgpCgogICAgIyBDcmVhdGUgdGhlIG91dHB1dCBkaXJlY3Rvcnk6CiAgICBvdXRwdXRfZGlyZWN0b3J5ID0gcGF0aGxpYi5QYXRoKG91dHB1dF9wYXRoKQogICAgaWYgbm90IG91dHB1dF9kaXJlY3RvcnkuZXhpc3RzKCk6CiAgICAgICAgb3V0cHV0X2RpcmVjdG9yeS5ta2RpcigpCgogICAgdHh0X2ZpbGVzX2RpcmVjdG9yeSA9IHBhdGhsaWIuUGF0aChpbnB1dF9wYXRoKQogICAgZXJyb3JzID0ge30KCiAgICByZXNfZGljdCA9IHt9CiAgICB0eHRfY29udGVudCA9IHt9CiAgICAjIExvYWQgdGhlIG1vZGVsOgogICAgdHJ5OgogICAgICAgIGFuYWx5emVyID0gX2dldF9hbmFseXplcl9lbmdpbmUobW9kZWwsIGVudGl0aWVzKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGVycm9yc1sibW9kZWwiXSA9IHN0cihlKQogICAgICAgIGxvZ2dlci5lcnJvcihmIkVycm9yIHdoZW4gZ2V0IHRoZSBtb2RlbDoge2V9IikKCiAgICBsb2dnZXIuaW5mbygiTW9kZWwgbG9hZGVkIikKICAgICMgR28gb3ZlciB0aGUgdGV4dCBmaWxlcyBpbiB0aGUgaW5wdXQgcGF0aCwgYW5hbHl6ZSBhbmQgYW5vbnltaXplIHRoZW06CiAgICBmb3IgaSwgdHh0X2ZpbGUgaW4gZW51bWVyYXRlKAogICAgICAgIHRxZG0oCiAgICAgICAgICAgIGxpc3QodHh0X2ZpbGVzX2RpcmVjdG9yeS5nbG9iKCIqLnR4dCIpKSwKICAgICAgICAgICAgZGVzYz0iUHJvY2Vzc2luZyBmaWxlcyIsCiAgICAgICAgICAgIHVuaXQ9ImZpbGUiLAogICAgICAgICkKICAgICk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICAjIExvYWQgdGhlIHN0ciBmcm9tIHRoZSB0ZXh0IGZpbGUKICAgICAgICAgICAgdGV4dCA9IHR4dF9maWxlLnJlYWRfdGV4dCgpCiAgICAgICAgICAgIHR4dF9jb250ZW50W3N0cih0eHRfZmlsZSldID0gdGV4dAogICAgICAgICAgICAjIFByb2Nlc3MgdGhlIHRleHQgdG8gcmVjb2dpbnplIHRoZSBwaWkgZW50aXRpZXMgaW4gaXQKICAgICAgICAgICAgYW5vbnltaXplZF90ZXh0LCByZXN1bHRzID0gX3Byb2Nlc3MoCiAgICAgICAgICAgICAgICB0ZXh0PXRleHQsCiAgICAgICAgICAgICAgICBtb2RlbD1hbmFseXplciwKICAgICAgICAgICAgICAgIGVudGl0aWVzPWVudGl0aWVzLAogICAgICAgICAgICAgICAgZW50aXRpZXNfb3BlcmF0b3JfbWFwPWVudGl0eV9vcGVyYXRvcl9tYXAsCiAgICAgICAgICAgICAgICBzY29yZV90aHJlc2hvbGQ9c2NvcmVfdGhyZXNob2xkLAogICAgICAgICAgICAgICAgaXNfZnVsbF90ZXh0PWlzX2Z1bGxfdGV4dCwKICAgICAgICAgICAgKQogICAgICAgICAgICByZXNfZGljdFtzdHIodHh0X2ZpbGUpXSA9IHJlc3VsdHMKICAgICAgICAgICAgIyBTdG9yZSB0aGUgYW5vbnltaXplZCB0ZXh0IGluIHRoZSBvdXRwdXQgcGF0aAogICAgICAgICAgICBvdXRwdXRfZmlsZSA9ICgKICAgICAgICAgICAgICAgIG91dHB1dF9kaXJlY3RvcnkKICAgICAgICAgICAgICAgIC8gZiJ7c3RyKHR4dF9maWxlLnJlbGF0aXZlX3RvKHR4dF9maWxlc19kaXJlY3RvcnkpKS5zcGxpdCgnLicpWzBdfS50eHQiCiAgICAgICAgICAgICkKICAgICAgICAgICAgb3V0cHV0X2ZpbGUucGFyZW50Lm1rZGlyKHBhcmVudHM9VHJ1ZSwgZXhpc3Rfb2s9VHJ1ZSkKICAgICAgICAgICAgd2l0aCBvcGVuKG91dHB1dF9maWxlLCAidyIpIGFzIGY6CiAgICAgICAgICAgICAgICBmLndyaXRlKGFub255bWl6ZWRfdGV4dCkKCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICBlcnJvcnNbc3RyKHR4dF9maWxlKV0gPSBzdHIoZSkKICAgICAgICAgICAgbG9nZ2VyLmVycm9yKGYiRXJyb3IgcHJvY2Vzc2luZyB7dHh0X2ZpbGV9OiB7ZX0iKQogICAgaWYgZ2VuZXJhdGVfaHRtbDoKICAgICAgICAjIEdlbmVyYXRlIHRoZSBodG1sIHJlcG9ydAogICAgICAgIGh0bWxfcmVzID0gX2dldF9hbGxfaHRtbCh0eHRfY29udGVudCwgcmVzX2RpY3QsIGlzX2Z1bGxfaHRtbCkKICAgICAgICAjIFN0b3JlIHRoZSBodG1sIHJlcG9ydCBpbiB0aGUgY29udGV4dAogICAgICAgIGFydGlfaHRtbCA9IG1scnVuLmFydGlmYWN0cy5BcnRpZmFjdChib2R5PWh0bWxfcmVzLCBmb3JtYXQ9Imh0bWwiLCBrZXk9aHRtbF9rZXkpCiAgICAgICAgY29udGV4dC5sb2dfYXJ0aWZhY3QoYXJ0aV9odG1sKQogICAgaWYgZ2VuZXJhdGVfanNvbjoKICAgICAgICAjIEdlbmVyYXRlIHRoZSBqc29uIHJlcG9ydAogICAgICAgIGpzb25fcmVzID0gX2dldF9hbGxfcnB0KHJlc19kaWN0LCBpc19mdWxsX3JlcG9ydCkKICAgICAgICByZXR1cm4gb3V0cHV0X3BhdGgsIGpzb25fcmVzLCBlcnJvcnMKICAgIHJldHVybiBvdXRwdXRfcGF0aCwgZXJyb3JzCg== + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG9zCmltcG9ydCBwYXRobGliCmltcG9ydCB0ZW1wZmlsZQppbXBvcnQgd2FybmluZ3MKZnJvbSB0eXBpbmcgaW1wb3J0IExpc3QsIFNldCwgVHVwbGUsIFVuaW9uCgppbXBvcnQgYW5ub3RhdGVkX3RleHQudXRpbCBhcyBhdF91dGlsCmltcG9ydCBtbHJ1bgppbXBvcnQgbmx0awppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBwcmVzaWRpb19hbmFseXplciBhcyBwYQppbXBvcnQgcHJlc2lkaW9fYW5vbnltaXplciBhcyBwcmVfYW5veW1pemVyCmZyb20gcHJlc2lkaW9fYW5vbnltaXplci5lbnRpdGllcyBpbXBvcnQgT3BlcmF0b3JDb25maWcKZnJvbSB0cWRtIGltcG9ydCB0cWRtCgp0cnk6CiAgICBpbXBvcnQgZmxhaXIgYXMgZmwKZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3I6CiAgICBwcmludCgiRmxhaXIgaXMgbm90IGluc3RhbGxlZCIpCgojIFRoZXJlIGlzIGEgY29uZmxpY3QgYmV0d2VlbiBSdXN0LWJhc2VkIHRva2VuaXplcnMnIHBhcmFsbGVsIHByb2Nlc3NpbmcKIyBhbmQgUHl0aG9uJ3MgZm9yayBvcGVyYXRpb25zIGR1cmluZyBtdWx0aXByb2Nlc3NpbmcuIFRvIGF2b2lkIHRoaXMsIHdlIG5lZWQKIyB0aGUgZm9sbG93aW5nIHR3byBsaW5lcwoKb3MuZW52aXJvblsiVE9LRU5JWkVSU19QQVJBTExFTElTTSJdID0gImZhbHNlIgp3YXJuaW5ncy5maWx0ZXJ3YXJuaW5ncygiaWdub3JlIikKCmxvZ2dlciA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCJwaWktcmVjb2duaXplciIpCgoKIyBBZGQgdGhlIGNvbnN0YW50IGNsYXNzZXMgb2YgTW9kZWxzIGFuZCBFbnRpdGllcyB0byBnb3Zlcm4gdGhlIHdob2xlIHBhY2thZ2UKY2xhc3MgTW9kZWxzOgogICAgV0hPTEUgPSAid2hvbGUiCiAgICBQQVRURVJOID0gInBhdHRlcm4iCiAgICBTUEFDWSA9ICJzcGFjeSIKICAgIEZMQUlSID0gImZsYWlyIgoKCmNsYXNzIEVudGl0aWVzOgogICAgQ1JFRElUX0NBUkQgPSAiQ1JFRElUX0NBUkQiCiAgICBTU04gPSAiU1NOIgogICAgUEhPTkUgPSAiUEhPTkUiCiAgICBFTUFJTCA9ICJFTUFJTCIKICAgIExPQ0FUSU9OID0gIkxPQ0FUSU9OIgogICAgUEVSU09OID0gIlBFUlNPTiIKICAgIE5SUCA9ICJOUlAiCiAgICBPUkdBTklaQVRJT04gPSAiT1JHQU5JWkFUSU9OIgogICAgREFURV9USU1FID0gIkRBVEVfVElNRSIKICAgIEdQRSA9ICgiR1BFIiwpCiAgICBNQUNfQUREUkVTUyA9ICJNQUNfQUREUkVTUyIKICAgIFVTX0JBTktfTlVNQkVSID0gIlVTX0JBTktfTlVNQkVSIgogICAgSU1FSSA9ICJJTUVJIgogICAgVElUTEUgPSAiVElUTEUiCiAgICBMSUNFTlNFX1BMQVRFID0gIkxJQ0VOU0VfUExBVEUiCiAgICBVU19QQVNTUE9SVCA9ICJVU19QQVNTUE9SVCIKICAgIENVUlJFTkNZID0gIkNVUlJFTkNZIgogICAgUk9VVElOR19OVU1CRVIgPSAiUk9VVElOR19OVU1CRVIiCiAgICBVU19JVElOID0gIlVTX0lUSU4iCiAgICBVU19CQU5LX05VTUJFUiA9ICJVU19CQU5LX05VTUJFUiIKICAgIFVTX0RSSVZFUl9MSUNFTlNFID0gIlVTX0RSSVZFUl9MSUNFTlNFIgogICAgQUdFID0gIkFHRSIKICAgIFBBU1NXT1JEID0gIlBBU1NXT1JEIgogICAgU1dJRlRfQ09ERSA9ICJTV0lGVF9DT0RFIgoKCmNsYXNzIFBhdHRlcm5SZWNvZ25pemVyRmFjdG9yeToKICAgICIiIgogICAgRmFjdG9yeSBmb3IgY3JlYXRpbmcgcGF0dGVybiByZWNvZ25pemVycywgaXQgY2FuIGJlIGV4dGVuZGVkIGluIHRoZSBmdXR1cmUgdG8KICAgIGFkZCBtb3JlIHJlZ2V4IHBhdHRlcm4gZm9yIGRpZmZlcmVudCBlbnRpdGllcy4gRm9yIHRoZSBwYXR0ZXJuIHJlY29nbml6ZXIgdG8gd29yaywKICAgIHdlIG5lZWQgY29uc3RydWN0IGEgbGlzdCBvZiByZWdleCBwYXR0ZXJucyBmb3IgZWFjaCBlbnRpdHkuCiAgICAiIiIKCiAgICBSRUNPR05JWkFCTEVfRU5USVRJRVMgPSB7CiAgICAgICAgIkNSRURJVF9DQVJEIjogW3BhLlBhdHRlcm4oIkNSRURJVF9DQVJEIiwgciJcYig/OlxkWyAtXSo/KXsxMywxNn1cYiIsIDAuNSldLAogICAgICAgICJTU04iOiBbcGEuUGF0dGVybigiU1NOIiwgciJcYlxkezN9LT9cZHsyfS0/XGR7NH1cYiIsIDAuNSldLAogICAgICAgICJQSE9ORSI6IFtwYS5QYXR0ZXJuKCJQSE9ORSIsIHIiXCg/XGR7M31cKT9bLS5cc10/XGR7M31bLS5cc10/XGR7NH0iLCAwLjUpXSwKICAgICAgICAiRU1BSUwiOiBbcGEuUGF0dGVybigiRU1BSUwiLCByIlxTK0BcUysiLCAwLjUpXSwKICAgIH0KCiAgICAjIGNyZWF0ZSBhIGxpc3Qgb2YgcGF0dGVybiByZWNvZ25pemVycwogICAgQGNsYXNzbWV0aG9kCiAgICBkZWYgX2NyZWF0ZV9wYXR0ZXJuX3JlY29nbml6ZXIoY2xzKToKICAgICAgICAiIiIKICAgICAgICBGb3IgZWFjaCBlbnRpdHksIGNyZWF0ZSBhIGxpc3Qgb2YgcGF0dGVybnMgdG8gcmVjb2duaXplIGl0CgogICAgICAgIDpwYXJhbSBjbHM6IFBhdHRlcm5SZWNvZ25pemVyRmFjdG9yeSBjbGFzcwoKICAgICAgICA6cmV0dXJuczogTGlzdCBvZiBwYXR0ZXJuIHJlY29nbml6ZXJzCiAgICAgICAgIiIiCgogICAgICAgICMgRW50aXRpZXMgdG8gcmVjb2duaXplIGFuZCB0aGVpciByZWdleCBwYXR0ZXJucwoKICAgICAgICByZXR1cm4gWwogICAgICAgICAgICBwYS5QYXR0ZXJuUmVjb2duaXplcihzdXBwb3J0ZWRfZW50aXR5PWVudGl0eSwgcGF0dGVybnM9cGF0dGVybikKICAgICAgICAgICAgZm9yIGVudGl0eSwgcGF0dGVybiBpbiBjbHMuUkVDT0dOSVpBQkxFX0VOVElUSUVTLml0ZW1zKCkKICAgICAgICBdCgoKY2xhc3MgQ3VzdG9tU3BhY3lSZWNvZ25pemVyKHBhLkxvY2FsUmVjb2duaXplcik6CiAgICAiIiIKICAgIEN1c3RvbSBTcGFjeSBSZWNvZ25pemVyIGZyb20gUHJlc2lkaW8gQW5hbHl6ZXIgdHJhaW5lZCBvbiBQcml2eSBkYXRhLgogICAgVGhlIHByaXZ5IGRhdGEgaXMgZ2VuZXJhdGVkIHVzaW5nIHRoaXMgaHR0cHM6Ly9naXRodWIuY29tL3BpeGllLWlvL3BpeGllL3RyZWUvbWFpbi9zcmMvZGF0YWdlbi9waWkvcHJpdnkKICAgIEl0IGNhbiBiZSB1c2VkIHRvIHJlY29nbml6ZSBjdXN0b20gZW50aXRpZXMsIFNpbmNlIHdlIHdhbnQgdG8gdXNlIFByZXNpZGlvJ3MgUmVnaXN0cmllcyB0byBnZW5lcmF0ZSBBbmFseXplckVuZ2luZSwKICAgIGl0IGluaGVyaXRzIGZyb20gUHJlc2lkaW8gQW5hbHl6ZXIncyBMb2NhbFJlY29nbml6ZXIgY2xhc3MuCiAgICAiIiIKCiAgICAjIEVudGl0aWVzIHRvIHJlY29nbml6ZQoKICAgIFJFQ09HTklaQUJMRV9FTlRJVElFUyA9IHsKICAgICAgICAiTE9DQVRJT04iLAogICAgICAgICJQRVJTT04iLAogICAgICAgICJOUlAiLAogICAgICAgICJPUkdBTklaQVRJT04iLAogICAgICAgICJEQVRFX1RJTUUiLAogICAgfQoKICAgICMgRGVmYXVsdCBleHBsYW5hdGlvbiBmb3IgdGhpcyByZWNvZ25pemVyCgogICAgX0RFRkFVTFRfRVhQTEFOQVRJT04gPSAoCiAgICAgICAgIklkZW50aWZpZWQgYXMge30gYnkgU3BhY3kncyBOYW1lZCBFbnRpdHkgUmVjb2duaXRpb24gKFByaXZ5LXRyYWluZWQpIgogICAgKQoKICAgICMgTGFiZWwgZ3JvdXBzIHRvIGNoZWNrCgogICAgX0RFRkFVTFRfQ0hFQ0tfTEFCRUxfR1JPVVBTID0gWwogICAgICAgICh7IkxPQ0FUSU9OIn0sIHsiTE9DIiwgIkxPQ0FUSU9OIiwgIlNUUkVFVF9BRERSRVNTIiwgIkNPT1JESU5BVEUifSksCiAgICAgICAgKHsiUEVSU09OIn0sIHsiUEVSIiwgIlBFUlNPTiJ9KSwKICAgICAgICAoeyJOUlAifSwgeyJOT1JQIiwgIk5SUCJ9KSwKICAgICAgICAoeyJPUkdBTklaQVRJT04ifSwgeyJPUkcifSksCiAgICAgICAgKHsiREFURV9USU1FIn0sIHsiREFURV9USU1FIn0pLAogICAgXQoKICAgICMgcHJldHJhaW5lZCBtb2RlbCBmb3IgdGhpcyByZWNvZ25pemVyCgogICAgX0RFRkFVTFRfTU9ERUxfTEFOR1VBR0VTID0gewogICAgICAgICJlbiI6ICJiZWtpL2VuX3NwYWN5X3BpaV9kaXN0aWxiZXJ0IiwKICAgIH0KCiAgICBfREVGQVVMVF9QUkVTSURJT19FUVVJVkFMRU5DRVMgPSB7CiAgICAgICAgIlBFUiI6ICJQRVJTT04iLAogICAgICAgICJMT0MiOiAiTE9DQVRJT04iLAogICAgICAgICJPUkciOiAiT1JHQU5JWkFUSU9OIiwKICAgICAgICAiTlJPUCI6ICJOUlAiLAogICAgICAgICJEQVRFX1RJTUUiOiAiREFURV9USU1FIiwKICAgIH0KCiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwKICAgICAgICBzdXBwb3J0ZWRfbGFuZ3VhZ2U6IHN0ciA9ICJlbiIsCiAgICAgICAgc3VwcG9ydGVkX2VudGl0aWVzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIGNoZWNrX2xhYmVsX2dyb3VwczogVHVwbGVbU2V0LCBTZXRdID0gTm9uZSwKICAgICAgICBjb250ZXh0OiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIG5lcl9zdHJlbmd0aDogZmxvYXQgPSAxLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIFNwYWN5IFJlY29nbml6ZXIuCgogICAgICAgIDpwYXJhbSBzdXBwb3J0ZWRfbGFuZ3VhZ2U6IExhbmd1YWdlIHRvIHVzZSwgZGVmYXVsdCBpcyBFbmdsaXNoCiAgICAgICAgOnBhcmFtIHN1cHBvcnRlZF9lbnRpdGllczogRW50aXRpZXMgdG8gdXNlIGZvciByZWNvZ25pdGlvbgogICAgICAgIDpwYXJhbSBjaGVja19sYWJlbF9ncm91cHM6IExhYmVsIGdyb3VwcyB0byBjaGVjayBmb3IgdGhlIGVudGl0aWVzCiAgICAgICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgQ29udGV4dCB0byB1c2UgaWYgYW55CiAgICAgICAgOnBhcmFtIG5lcl9zdHJlbmd0aDogICAgICAgRGVmYXVsdCBjb25maWRlbmNlIGZvciBORVIgcHJlZGljdGlvbgoKICAgICAgICA6cmV0dXJuczogU3BhY3lSZWNvZ25pemVyIG9iamVjdAogICAgICAgICIiIgoKICAgICAgICAjIERlZmF1bHQgY29uZmlkZW5jZSBmb3IgTkVSIHByZWRpY3Rpb24KICAgICAgICBzZWxmLm5lcl9zdHJlbmd0aCA9IG5lcl9zdHJlbmd0aAoKICAgICAgICBzZWxmLmNoZWNrX2xhYmVsX2dyb3VwcyA9IGNoZWNrX2xhYmVsX2dyb3VwcyBvciBzZWxmLl9ERUZBVUxUX0NIRUNLX0xBQkVMX0dST1VQUwogICAgICAgIHN1cHBvcnRlZF9lbnRpdGllcyA9IHN1cHBvcnRlZF9lbnRpdGllcyBvciBzZWxmLlJFQ09HTklaQUJMRV9FTlRJVElFUwogICAgICAgIHN1cGVyKCkuX19pbml0X18oCiAgICAgICAgICAgIHN1cHBvcnRlZF9lbnRpdGllcz1zdXBwb3J0ZWRfZW50aXRpZXMsCiAgICAgICAgICAgIHN1cHBvcnRlZF9sYW5ndWFnZT1zdXBwb3J0ZWRfbGFuZ3VhZ2UsCiAgICAgICAgKQoKICAgICMgZ2V0IHRoZSBwcmVzaWRpbyBleHBsYW5hdGlvbiBmb3IgdGhlIHJlc3VsdAoKICAgIGRlZiBfYnVpbGRfc3BhY3lfZXhwbGFuYXRpb24oCiAgICAgICAgc2VsZiwgb3JpZ2luYWxfc2NvcmU6IGZsb2F0LCBleHBsYW5hdGlvbjogc3RyCiAgICApIC0+IHBhLkFuYWx5c2lzRXhwbGFuYXRpb246CiAgICAgICAgIiIiCiAgICAgICAgQ3JlYXRlIGV4cGxhbmF0aW9uIGZvciB3aHkgdGhpcyByZXN1bHQgd2FzIGRldGVjdGVkLgoKICAgICAgICA6cGFyYW0gb3JpZ2luYWxfc2NvcmU6IFNjb3JlIGdpdmVuIGJ5IHRoaXMgcmVjb2duaXplcgogICAgICAgIDpwYXJhbSBleHBsYW5hdGlvbjogICAgRXhwbGFuYXRpb24gc3RyaW5nCgogICAgICAgIDpyZXR1cm5zOiBQcmVzaWRpbyBBbmFseXNpc0V4cGxhbmF0aW9uIG9iamVjdAogICAgICAgICIiIgogICAgICAgIGV4cGxhbmF0aW9uID0gcGEuQW5hbHlzaXNFeHBsYW5hdGlvbigKICAgICAgICAgICAgcmVjb2duaXplcj1zZWxmLl9fY2xhc3NfXy5fX25hbWVfXywKICAgICAgICAgICAgb3JpZ2luYWxfc2NvcmU9b3JpZ2luYWxfc2NvcmUsCiAgICAgICAgICAgIHRleHR1YWxfZXhwbGFuYXRpb249ZXhwbGFuYXRpb24sCiAgICAgICAgKQogICAgICAgIHJldHVybiBleHBsYW5hdGlvbgoKICAgICMgbWFpbiBtZXRob2QgZm9yIHRoZSByZWNvZ25pemVyCiAgICBkZWYgYW5hbHl6ZShzZWxmLCB0ZXh0OiBzdHIsIGVudGl0aWVzOiBMaXN0W3N0cl0sIG5scF9hcnRpZmFjdHM9Tm9uZSk6ICAjIG5vcWEgRDEwMgogICAgICAgICIiIgogICAgICAgIEFuYWx5emUgdGV4dCB1c2luZyBTcGFjeS4KCiAgICAgICAgOnBhcmFtIHRleHQ6ICAgICAgICAgIFRleHQgdG8gYW5hbHl6ZQogICAgICAgIDpwYXJhbSBlbnRpdGllczogICAgICBFbnRpdGllcyB0byBhbmFseXplCiAgICAgICAgOnBhcmFtIG5scF9hcnRpZmFjdHM6IE5MUCBhcnRpZmFjdHMgdG8gdXNlCgogICAgICAgIDpyZXR1cm5zOiBMaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgb2JqZWN0cwogICAgICAgICIiIgogICAgICAgIHJlc3VsdHMgPSBbXQogICAgICAgIGlmIG5vdCBubHBfYXJ0aWZhY3RzOgogICAgICAgICAgICBsb2dnZXIud2FybmluZygiU2tpcHBpbmcgU3BhQ3ksIG5scCBhcnRpZmFjdHMgbm90IHByb3ZpZGVkLi4uIikKICAgICAgICAgICAgcmV0dXJuIHJlc3VsdHMKCiAgICAgICAgbmVyX2VudGl0aWVzID0gbmxwX2FydGlmYWN0cy5lbnRpdGllcwoKICAgICAgICAjIHJlY29nbml6ZSB0aGUgc3VwcG9ydGVkIGVudGl0aWVzCiAgICAgICAgZm9yIGVudGl0eSBpbiBlbnRpdGllczoKICAgICAgICAgICAgaWYgZW50aXR5IG5vdCBpbiBzZWxmLnN1cHBvcnRlZF9lbnRpdGllczoKICAgICAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgICAgIGZvciBlbnQgaW4gbmVyX2VudGl0aWVzOgogICAgICAgICAgICAgICAgaWYgbm90IHNlbGYuX19jaGVja19sYWJlbChlbnRpdHksIGVudC5sYWJlbF8sIHNlbGYuY2hlY2tfbGFiZWxfZ3JvdXBzKToKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQoKICAgICAgICAgICAgICAgICMgc3RyaW5nIG9mIHRoZSBleHBsYW5hdGlvbiBzYXlpbmcgdGhlIGVudGl0eSBpcyByZWNvZ25pemVkIGJ5IHNwYWN5CiAgICAgICAgICAgICAgICB0ZXh0dWFsX2V4cGxhbmF0aW9uID0gc2VsZi5fREVGQVVMVF9FWFBMQU5BVElPTi5mb3JtYXQoZW50LmxhYmVsXykKICAgICAgICAgICAgICAgIGV4cGxhbmF0aW9uID0gc2VsZi5fYnVpbGRfc3BhY3lfZXhwbGFuYXRpb24oCiAgICAgICAgICAgICAgICAgICAgc2VsZi5uZXJfc3RyZW5ndGgsIHRleHR1YWxfZXhwbGFuYXRpb24KICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAjIGNyZWF0ZSB0aGUgc3RhbmRhcmQgcmVzdWx0IHdpdGggdGhlIGVudGl0eSwgc3RhcnQsIGVuZCwgc2NvcmUsIGFuZCBleHBsYW5hdGlvbgogICAgICAgICAgICAgICAgc3BhY3lfcmVzdWx0ID0gcGEuUmVjb2duaXplclJlc3VsdCgKICAgICAgICAgICAgICAgICAgICBlbnRpdHlfdHlwZT1lbnRpdHksCiAgICAgICAgICAgICAgICAgICAgc3RhcnQ9ZW50LnN0YXJ0X2NoYXIsCiAgICAgICAgICAgICAgICAgICAgZW5kPWVudC5lbmRfY2hhciwKICAgICAgICAgICAgICAgICAgICBzY29yZT1zZWxmLm5lcl9zdHJlbmd0aCwKICAgICAgICAgICAgICAgICAgICBhbmFseXNpc19leHBsYW5hdGlvbj1leHBsYW5hdGlvbiwKICAgICAgICAgICAgICAgICAgICByZWNvZ25pdGlvbl9tZXRhZGF0YT17CiAgICAgICAgICAgICAgICAgICAgICAgIHBhLlJlY29nbml6ZXJSZXN1bHQuUkVDT0dOSVpFUl9OQU1FX0tFWTogc2VsZi5uYW1lCiAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKHNwYWN5X3Jlc3VsdCkKCiAgICAgICAgcmV0dXJuIHJlc3VsdHMKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgX19jaGVja19sYWJlbCgKICAgICAgICBlbnRpdHk6IHN0ciwgbGFiZWw6IHN0ciwgY2hlY2tfbGFiZWxfZ3JvdXBzOiBUdXBsZVtTZXQsIFNldF0KICAgICkgLT4gYm9vbDoKICAgICAgICAiIiIKICAgICAgICBDaGVjayBpZiB0aGUgbGFiZWwgaXMgaW4gdGhlIGxhYmVsIGdyb3VwLgoKICAgICAgICA6cGFyYW0gZW50aXR5OiAgICAgICAgICAgICBFbnRpdHkgdG8gY2hlY2sKICAgICAgICA6cGFyYW0gbGFiZWw6ICAgICAgICAgICAgICBMYWJlbCB0byBjaGVjawogICAgICAgIDpwYXJhbSBjaGVja19sYWJlbF9ncm91cHM6IExhYmVsIGdyb3VwcyB0byBjaGVjawoKICAgICAgICA6cmV0dXJuczogVHJ1ZSBpZiB0aGUgbGFiZWwgaXMgaW4gdGhlIGxhYmVsIGdyb3VwLCBGYWxzZSBvdGhlcndpc2UKICAgICAgICAiIiIKICAgICAgICByZXR1cm4gYW55KAogICAgICAgICAgICBlbnRpdHkgaW4gZWdycCBhbmQgbGFiZWwgaW4gbGdycCBmb3IgZWdycCwgbGdycCBpbiBjaGVja19sYWJlbF9ncm91cHMKICAgICAgICApCgoKIyBDbGFzcyB0byB1c2UgRmxhaXIgd2l0aCBQcmVzaWRpbyBhcyBhbiBleHRlcm5hbCByZWNvZ25pemVyLgpjbGFzcyBGbGFpclJlY29nbml6ZXIocGEuRW50aXR5UmVjb2duaXplcik6CiAgICAiIiIKICAgIFdyYXBwZXIgZm9yIGEgZmxhaXIgbW9kZWwsIGlmIG5lZWRlZCB0byBiZSB1c2VkIHdpdGhpbiBQcmVzaWRpbyBBbmFseXplci4KICAgIFRoaXMgaXMgdG8gbWFrZSBzdXJlIHRoZSByZWNvZ25pemVyIGNhbiBiZSByZWdpc3RlcmVkIHdpdGggUHJlc2lkaW8gcmVnaXN0cnkuCiAgICAiIiIKCiAgICBSRUNPR05JWkFCTEVfRU5USVRJRVMgPSB7CiAgICAgICAgIkxPQ0FUSU9OIiwKICAgICAgICAiUEVSU09OIiwKICAgICAgICAiTlJQIiwKICAgICAgICAiR1BFIiwKICAgICAgICAiT1JHQU5JWkFUSU9OIiwKICAgICAgICAiTUFDX0FERFJFU1MiLAogICAgICAgICJVU19CQU5LX05VTUJFUiIsCiAgICAgICAgIklNRUkiLAogICAgICAgICJUSVRMRSIsCiAgICAgICAgIkxJQ0VOU0VfUExBVEUiLAogICAgICAgICJVU19QQVNTUE9SVCIsCiAgICAgICAgIkNVUlJFTkNZIiwKICAgICAgICAiUk9VVElOR19OVU1CRVIiLAogICAgICAgICJVU19JVElOIiwKICAgICAgICAiVVNfQkFOS19OVU1CRVIiLAogICAgICAgICJVU19EUklWRVJfTElDRU5TRSIsCiAgICAgICAgIkFHRSIsCiAgICAgICAgIlBBU1NXT1JEIiwKICAgICAgICAiU1dJRlRfQ09ERSIsCiAgICB9CgogICAgIyBUaGlzIGlzIHVzZWQgdG8gY29uc3RydWN0IHRoZSBleHBsYW5hdGlvbiBmb3IgdGhlIHJlc3VsdAoKICAgIF9ERUZBVUxUX0VYUExBTkFUSU9OID0gIklkZW50aWZpZWQgYXMge30gYnkgRmxhaXIncyBOYW1lZCBFbnRpdHkgUmVjb2duaXRpb24iCgogICAgX0RFRkFVTFRfQ0hFQ0tfTEFCRUxfR1JPVVBTID0gWwogICAgICAgICh7IkxPQ0FUSU9OIn0sIHsiTE9DIiwgIkxPQ0FUSU9OIiwgIlNUUkVFVF9BRERSRVNTIiwgIkNPT1JESU5BVEUifSksCiAgICAgICAgKHsiUEVSU09OIn0sIHsiUEVSIiwgIlBFUlNPTiJ9KSwKICAgICAgICAoeyJOUlAifSwgeyJOT1JQIiwgIk5SUCJ9KSwKICAgICAgICAoeyJHUEUifSwgeyJHUEUifSksCiAgICAgICAgKHsiT1JHQU5JWkFUSU9OIn0sIHsiT1JHIn0pLAogICAgICAgICh7Ik1BQ19BRERSRVNTIn0sIHsiTUFDX0FERFJFU1MifSksCiAgICAgICAgKHsiVVNfQkFOS19OVU1CRVIifSwgeyJVU19CQU5LX05VTUJFUiJ9KSwKICAgICAgICAoeyJJTUVJIn0sIHsiSU1FSSJ9KSwKICAgICAgICAoeyJUSVRMRSJ9LCB7IlRJVExFIn0pLAogICAgICAgICh7IkxJQ0VOU0VfUExBVEUifSwgeyJMSUNFTlNFX1BMQVRFIn0pLAogICAgICAgICh7IlVTX1BBU1NQT1JUIn0sIHsiVVNfUEFTU1BPUlQifSksCiAgICAgICAgKHsiQ1VSUkVOQ1kifSwgeyJDVVJSRU5DWSJ9KSwKICAgICAgICAoeyJST1VUSU5HX05VTUJFUiJ9LCB7IlJPVVRJTkdfTlVNQkVSIn0pLAogICAgICAgICh7IkFHRSJ9LCB7IkFHRSJ9KSwKICAgICAgICAoeyJDVVJSRU5DWSJ9LCB7IkNVUlJFTkNZIn0pLAogICAgICAgICh7IlNXSUZUX0NPREUifSwgeyJTV0lGVF9DT0RFIn0pLAogICAgICAgICh7IlVTX0lUSU4ifSwgeyJVU19JVElOIn0pLAogICAgICAgICh7IlVTX0JBTktfTlVNQkVSIn0sIHsiVVNfQkFOS19OVU1CRVIifSksCiAgICAgICAgKHsiVVNfRFJJVkVSX0xJQ0VOU0UifSwgeyJVU19EUklWRVJfTElDRU5TRSJ9KSwKICAgIF0KCiAgICBfREVGQVVMVF9NT0RFTF9MQU5HVUFHRVMgPSB7CiAgICAgICAgImVuIjogImJla2kvZmxhaXItcGlpLWRpc3RpbGJlcnQiLAogICAgfQoKICAgIF9ERUZBVUxUX1BSRVNJRElPX0VRVUlWQUxFTkNFUyA9IHsKICAgICAgICAiUEVSIjogIlBFUlNPTiIsCiAgICAgICAgIkxPQyI6ICJMT0NBVElPTiIsCiAgICAgICAgIk9SRyI6ICJPUkdBTklaQVRJT04iLAogICAgICAgICJOUk9QIjogIk5SUCIsCiAgICAgICAgIlVSTCI6ICJVUkwiLAogICAgICAgICJVU19JVElOIjogIlVTX0lUSU4iLAogICAgICAgICJVU19QQVNTUE9SVCI6ICJVU19QQVNTUE9SVCIsCiAgICAgICAgIklCQU5fQ09ERSI6ICJJQkFOX0NPREUiLAogICAgICAgICJJUF9BRERSRVNTIjogIklQX0FERFJFU1MiLAogICAgICAgICJFTUFJTF9BRERSRVNTIjogIkVNQUlMIiwKICAgICAgICAiVVNfRFJJVkVSX0xJQ0VOU0UiOiAiVVNfRFJJVkVSX0xJQ0VOU0UiLAogICAgICAgICJVU19CQU5LX05VTUJFUiI6ICJVU19CQU5LX05VTUJFUiIsCiAgICB9CgogICAgZGVmIF9faW5pdF9fKAogICAgICAgIHNlbGYsCiAgICAgICAgc3VwcG9ydGVkX2xhbmd1YWdlOiBzdHIgPSAiZW4iLAogICAgICAgIHN1cHBvcnRlZF9lbnRpdGllczogTGlzdFtzdHJdID0gTm9uZSwKICAgICAgICBjaGVja19sYWJlbF9ncm91cHM6IFR1cGxlW1NldCwgU2V0XSA9IE5vbmUsCiAgICApOgogICAgICAgICIiIgogICAgICAgIEluaXRpYWxpemUgdGhlIEZsYWlyUmVjb2duaXplci4KCiAgICAgICAgOnBhcmFtIHN1cHBvcnRlZF9sYW5ndWFnZTogTGFuZ3VhZ2UgdG8gdXNlCiAgICAgICAgOnBhcmFtIHN1cHBvcnRlZF9lbnRpdGllczogRW50aXRpZXMgdG8gdXNlCiAgICAgICAgOnBhcmFtIGNoZWNrX2xhYmVsX2dyb3VwczogTGFiZWwgZ3JvdXBzIHRvIGNoZWNrCgogICAgICAgIDpyZXR1cm5zOiBGbGFpclJlY29nbml6ZXIgb2JqZWN0CgogICAgICAgICIiIgogICAgICAgIHNlbGYuY2hlY2tfbGFiZWxfZ3JvdXBzID0gY2hlY2tfbGFiZWxfZ3JvdXBzIG9yIHNlbGYuX0RFRkFVTFRfQ0hFQ0tfTEFCRUxfR1JPVVBTCgogICAgICAgIHN1cHBvcnRlZF9lbnRpdGllcyA9IHN1cHBvcnRlZF9lbnRpdGllcyBvciBzZWxmLlJFQ09HTklaQUJMRV9FTlRJVElFUwogICAgICAgIHNlbGYubW9kZWwgPSBmbC5tb2RlbHMuU2VxdWVuY2VUYWdnZXIubG9hZCgKICAgICAgICAgICAgc2VsZi5fREVGQVVMVF9NT0RFTF9MQU5HVUFHRVMuZ2V0KHN1cHBvcnRlZF9sYW5ndWFnZSkKICAgICAgICApCgogICAgICAgIHN1cGVyKCkuX19pbml0X18oCiAgICAgICAgICAgIHN1cHBvcnRlZF9lbnRpdGllcz1zdXBwb3J0ZWRfZW50aXRpZXMsCiAgICAgICAgICAgIHN1cHBvcnRlZF9sYW5ndWFnZT1zdXBwb3J0ZWRfbGFuZ3VhZ2UsCiAgICAgICAgICAgIG5hbWU9IkZsYWlyIEFuYWx5dGljcyIsCiAgICAgICAgKQoKICAgICMgbWFpbiBtZXRob2QgZm9yIHRoZSByZWNvZ25pemVyCiAgICBkZWYgYW5hbHl6ZSgKICAgICAgICBzZWxmLAogICAgICAgIHRleHQ6IHN0ciwKICAgICAgICBlbnRpdGllczogTGlzdFtzdHJdLAogICAgICAgIG5scF9hcnRpZmFjdHM6IHBhLm5scF9lbmdpbmUuTmxwQXJ0aWZhY3RzID0gTm9uZSwKICAgICkgLT4gTGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XToKICAgICAgICAiIiIKICAgICAgICBBbmFseXplIHRleHQgYW5kIHJldHVybiB0aGUgcmVzdWx0cy4KCiAgICAgICAgOnBhcmFtIHRleHQ6ICAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgICAgICA6cGFyYW0gZW50aXRpZXM6ICAgICAgVGhlIGxpc3Qgb2YgZW50aXRpZXMgdG8gcmVjb2duaXplLgogICAgICAgIDpwYXJhbSBubHBfYXJ0aWZhY3RzOiBOb3QgdXNlZCBieSB0aGlzIHJlY29nbml6ZXIgYnV0IG5lZWRlZCBmb3IgdGhlIGludGVyZmFjZS4KCiAgICAgICAgOnJldHVybnM6IFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSB0aGUgcmVjb2duaXplZCBGbGFpciBkZXRlY3Rpb25zLgogICAgICAgICIiIgoKICAgICAgICByZXN1bHRzID0gW10KCiAgICAgICAgc2VudGVuY2VzID0gZmwuZGF0YS5TZW50ZW5jZSh0ZXh0KQogICAgICAgIHNlbGYubW9kZWwucHJlZGljdChzZW50ZW5jZXMpCgogICAgICAgICMgSWYgdGhlcmUgYXJlIG5vIHNwZWNpZmljIGxpc3Qgb2YgZW50aXRpZXMsIHdlIHdpbGwgbG9vayBmb3IgYWxsIG9mIGl0LgogICAgICAgIGlmIG5vdCBlbnRpdGllczoKICAgICAgICAgICAgZW50aXRpZXMgPSBzZWxmLnN1cHBvcnRlZF9lbnRpdGllcwoKICAgICAgICAjIEdvIG92ZXIgdGhlIGVudGl0aWVzIGFuZCBjaGVjayBpZiB0aGV5IGFyZSBpbiB0aGUgc3VwcG9ydGVkIGVudGl0aWVzIGxpc3QuCiAgICAgICAgZm9yIGVudGl0eSBpbiBlbnRpdGllczoKICAgICAgICAgICAgaWYgZW50aXR5IG5vdCBpbiBzZWxmLnN1cHBvcnRlZF9lbnRpdGllczoKICAgICAgICAgICAgICAgIGNvbnRpbnVlCgogICAgICAgICAgICAjIEdvIG92ZXIgdGhlIHNlbnRlbmNlcyBhbmQgY2hlY2sgaWYgdGhlIGVudGl0eSBpcyBpbiB0aGUgc2VudGVuY2UuCiAgICAgICAgICAgIGZvciBlbnQgaW4gc2VudGVuY2VzLmdldF9zcGFucygibmVyIik6CiAgICAgICAgICAgICAgICBpZiBub3Qgc2VsZi5fX2NoZWNrX2xhYmVsKAogICAgICAgICAgICAgICAgICAgIGVudGl0eSwgZW50LmxhYmVsc1swXS52YWx1ZSwgc2VsZi5jaGVja19sYWJlbF9ncm91cHMKICAgICAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKCiAgICAgICAgICAgICAgICAjIElmIHRoZSBlbnRpdHkgaXMgaW4gdGhlIHNlbnRlbmNlLCB3ZSB3aWxsIGFkZCBpdCB0byB0aGUgcmVzdWx0cy4KICAgICAgICAgICAgICAgIHRleHR1YWxfZXhwbGFuYXRpb24gPSBzZWxmLl9ERUZBVUxUX0VYUExBTkFUSU9OLmZvcm1hdCgKICAgICAgICAgICAgICAgICAgICBlbnQubGFiZWxzWzBdLnZhbHVlCiAgICAgICAgICAgICAgICApCgogICAgICAgICAgICAgICAgIyBCdWlsZCB0aGUgZXhwbGFuYXRpb24gZm9yIHRoZSByZXN1bHQKICAgICAgICAgICAgICAgIGV4cGxhbmF0aW9uID0gc2VsZi5fYnVpbGRfZmxhaXJfZXhwbGFuYXRpb24oCiAgICAgICAgICAgICAgICAgICAgcm91bmQoZW50LnNjb3JlLCAyKSwgdGV4dHVhbF9leHBsYW5hdGlvbgogICAgICAgICAgICAgICAgKQoKICAgICAgICAgICAgICAgIGZsYWlyX3Jlc3VsdCA9IHNlbGYuX2NvbnZlcnRfdG9fcmVjb2duaXplcl9yZXN1bHQoZW50LCBleHBsYW5hdGlvbikKCiAgICAgICAgICAgICAgICByZXN1bHRzLmFwcGVuZChmbGFpcl9yZXN1bHQpCgogICAgICAgIHJldHVybiByZXN1bHRzCgogICAgZGVmIF9jb252ZXJ0X3RvX3JlY29nbml6ZXJfcmVzdWx0KAogICAgICAgIHNlbGYsIGVudGl0eTogZmwuZGF0YS5TcGFuLCBleHBsYW5hdGlvbjogc3RyCiAgICApIC0+IHBhLlJlY29nbml6ZXJSZXN1bHQ6CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCBGbGFpciByZXN1bHQgdG8gUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdC4KCiAgICAgICAgOnBhcmFtIGVudGl0eTogICAgICBGbGFpciBlbnRpdHkgb2YgU3BhbgogICAgICAgIDpwYXJhbSBleHBsYW5hdGlvbjogUHJlc2lkaW8gQW5hbHlzaXNFeHBsYW5hdGlvbgoKICAgICAgICA6cmV0dXJuczogUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdAogICAgICAgICIiIgoKICAgICAgICAjIENvbnZlcnQgdGhlIGVudGl0eSB0eXBlIHRvIFByZXNpZGlvIGVudGl0eSB0eXBlCiAgICAgICAgZW50aXR5X3R5cGUgPSBzZWxmLl9ERUZBVUxUX1BSRVNJRElPX0VRVUlWQUxFTkNFUy5nZXQoZW50aXR5LnRhZywgZW50aXR5LnRhZykKCiAgICAgICAgIyBDb252ZXJ0IHRoZSBzY29yZSB0byBQcmVzaWRpbyBzY29yZQogICAgICAgIGZsYWlyX3Njb3JlID0gcm91bmQoZW50aXR5LnNjb3JlLCAyKQoKICAgICAgICAjIENyZWF0ZSB0aGUgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBmcm9tIHRoZSBGbGFpciBlbnRpdHkKICAgICAgICBmbGFpcl9yZXN1bHRzID0gcGEuUmVjb2duaXplclJlc3VsdCgKICAgICAgICAgICAgZW50aXR5X3R5cGU9ZW50aXR5X3R5cGUsCiAgICAgICAgICAgIHN0YXJ0PWVudGl0eS5zdGFydF9wb3NpdGlvbiwKICAgICAgICAgICAgZW5kPWVudGl0eS5lbmRfcG9zaXRpb24sCiAgICAgICAgICAgIHNjb3JlPWZsYWlyX3Njb3JlLAogICAgICAgICAgICBhbmFseXNpc19leHBsYW5hdGlvbj1leHBsYW5hdGlvbiwKICAgICAgICApCgogICAgICAgIHJldHVybiBmbGFpcl9yZXN1bHRzCgogICAgZGVmIF9idWlsZF9mbGFpcl9leHBsYW5hdGlvbigKICAgICAgICBzZWxmLCBvcmlnaW5hbF9zY29yZTogZmxvYXQsIGV4cGxhbmF0aW9uOiBzdHIKICAgICkgLT4gcGEuQW5hbHlzaXNFeHBsYW5hdGlvbjoKICAgICAgICAiIiIKICAgICAgICBDcmVhdGUgZXhwbGFuYXRpb24gZm9yIHdoeSB0aGlzIHJlc3VsdCB3YXMgZGV0ZWN0ZWQuCgogICAgICAgIDpwYXJhbSBvcmlnaW5hbF9zY29yZTogU2NvcmUgZ2l2ZW4gYnkgdGhpcyByZWNvZ25pemVyCiAgICAgICAgOnBhcmFtIGV4cGxhbmF0aW9uOiAgICBFeHBsYW5hdGlvbiBzdHJpbmcKCiAgICAgICAgOnJldHVybnM6IFByZXNpZGlvIEFuYWx5c2lzRXhwbGFuYXRpb24KICAgICAgICAiIiIKCiAgICAgICAgIyBDcmVhdGUgdGhlIFByZXNpZGlvIEFuYWx5c2lzRXhwbGFuYXRpb24gZm9yIHRoZSByZXN1bHQKICAgICAgICBleHBsYW5hdGlvbiA9IHBhLkFuYWx5c2lzRXhwbGFuYXRpb24oCiAgICAgICAgICAgIHJlY29nbml6ZXI9c2VsZi5fX2NsYXNzX18uX19uYW1lX18sCiAgICAgICAgICAgIG9yaWdpbmFsX3Njb3JlPW9yaWdpbmFsX3Njb3JlLAogICAgICAgICAgICB0ZXh0dWFsX2V4cGxhbmF0aW9uPWV4cGxhbmF0aW9uLAogICAgICAgICkKICAgICAgICByZXR1cm4gZXhwbGFuYXRpb24KCiAgICAjIHNhbml0eSBjaGVjayBvZiB0aGUgZW50aXR5IGFuZCBsYWJlbCBiZWZvcmUgcmVjb2duaXRpb24KICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiBfX2NoZWNrX2xhYmVsKAogICAgICAgIGVudGl0eTogc3RyLCBsYWJlbDogc3RyLCBjaGVja19sYWJlbF9ncm91cHM6IFR1cGxlW1NldCwgU2V0XQogICAgKSAtPiBib29sOgogICAgICAgIHJldHVybiBhbnkoCiAgICAgICAgICAgIGVudGl0eSBpbiBlZ3JwIGFuZCBsYWJlbCBpbiBsZ3JwIGZvciBlZ3JwLCBsZ3JwIGluIGNoZWNrX2xhYmVsX2dyb3VwcwogICAgICAgICkKCgojIGdldCB0aGUgYW5hbHl6ZXIgZW5naW5lIGJhc2VkIG9uIHRoZSBtb2RlbApkZWYgX2dldF9hbmFseXplcl9lbmdpbmUoCiAgICBtb2RlbDogc3RyID0gTm9uZSwgZW50aXRpZXM6IExpc3Rbc3RyXSA9IE5vbmUKKSAtPiBwYS5BbmFseXplckVuZ2luZToKICAgICIiIgogICAgUmV0dXJuIHBhLkFuYWx5emVyRW5naW5lLgoKICAgIDpwYXJhbSBtb2RlbDogVGhlIG1vZGVsIHRvIHVzZS4gQ2FuIGJlICJzcGFjeSIsICJmbGFpciIsICJwYXR0ZXJuIiBvciAid2hvbGUiLgogICAgOnBhcmFtIGVudGl0aWVzOiBUaGUgbGlzdCBvZiBlbnRpdGllcyB0byB1c2UuCgogICAgOnJldHVybnM6IHBhLkFuYWx5emVyRW5naW5lCiAgICAiIiIKICAgICMgcmVjb2duaXplciByZWdpc3RyeSB0aGF0IGNhbiBzdG9yZSBtdWx0aXBsZSByZWNvZ25pemVycwogICAgcmVnaXN0cnkgPSBwYS5SZWNvZ25pemVyUmVnaXN0cnkoKQogICAgaWYgbW9kZWwgPT0gTW9kZWxzLlNQQUNZOgogICAgICAgICMgY3VzdG9tIHNwYWN5IHJlY29nbml6ZXIKICAgICAgICBzcGFjeV9yZWNvZ25pemVyID0gQ3VzdG9tU3BhY3lSZWNvZ25pemVyKCkKICAgICAgICAjIGFkZCB0aGUgY3VzdG9tIGJ1aWxkIHNwYWN5IHJlY29nbml6ZXIKICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihzcGFjeV9yZWNvZ25pemVyKQogICAgZWxpZiBtb2RlbCA9PSBNb2RlbHMuRkxBSVI6CiAgICAgICAgIyBwcmUtdHJhaW5lZCBmbGFpciByZWNvZ25pemVyCiAgICAgICAgZmxhaXJfcmVjb2duaXplciA9IEZsYWlyUmVjb2duaXplcigpCiAgICAgICAgIyBhZGQgdGhlIGN1c3RvbSBidWlsZCBmbGFpciByZWNvZ25pemVyCiAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIoZmxhaXJfcmVjb2duaXplcikKICAgIGVsaWYgbW9kZWwgPT0gTW9kZWxzLlBBVFRFUk46CiAgICAgICAgIyBhZGQgdGhlIHBhdHRlcm4gcmVjb2duaXplcgogICAgICAgIHBhdHRlcm5fcmVjb2duaXplcl9mYWN0b3J5ID0gUGF0dGVyblJlY29nbml6ZXJGYWN0b3J5KCkKICAgICAgICBmb3IgcmVjb2duaXplciBpbiBwYXR0ZXJuX3JlY29nbml6ZXJfZmFjdG9yeS5fY3JlYXRlX3BhdHRlcm5fcmVjb2duaXplcigpOgogICAgICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihyZWNvZ25pemVyKQogICAgZWxpZiBtb2RlbCA9PSBNb2RlbHMuV0hPTEU6CiAgICAgICAgc3BhY3lfcmVjb2duaXplciA9IEN1c3RvbVNwYWN5UmVjb2duaXplcigpCiAgICAgICAgZmxhaXJfcmVjb2duaXplciA9IEZsYWlyUmVjb2duaXplcigpCiAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIoc3BhY3lfcmVjb2duaXplcikKICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihmbGFpcl9yZWNvZ25pemVyKQogICAgICAgICMgYWRkIHRoZSBwYXR0ZXJuIHJlY29nbml6ZXIKICAgICAgICBwYXR0ZXJuX3JlY29nbml6ZXJfZmFjdG9yeSA9IFBhdHRlcm5SZWNvZ25pemVyRmFjdG9yeSgpCiAgICAgICAgZm9yIHJlY29nbml6ZXIgaW4gcGF0dGVybl9yZWNvZ25pemVyX2ZhY3RvcnkuX2NyZWF0ZV9wYXR0ZXJuX3JlY29nbml6ZXIoKToKICAgICAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIocmVjb2duaXplcikKICAgIGVsaWYgbm90IG1vZGVsIGFuZCBlbnRpdGllczoKICAgICAgICBpZiBzZXQoZW50aXRpZXMpICYgQ3VzdG9tU3BhY3lSZWNvZ25pemVyLlJFQ09HTklaQUJMRV9FTlRJVElFUzoKICAgICAgICAgICAgc3BhY3lfcmVjb2duaXplciA9IEN1c3RvbVNwYWN5UmVjb2duaXplcigpCiAgICAgICAgICAgIHJlZ2lzdHJ5LmFkZF9yZWNvZ25pemVyKHNwYWN5X3JlY29nbml6ZXIpCiAgICAgICAgaWYgc2V0KGVudGl0aWVzKSAmIEZsYWlyUmVjb2duaXplci5SRUNPR05JWkFCTEVfRU5USVRJRVM6CiAgICAgICAgICAgIGZsYWlyX3JlY29nbml6ZXIgPSBGbGFpclJlY29nbml6ZXIoKQogICAgICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihmbGFpcl9yZWNvZ25pemVyKQogICAgICAgICMgYWRkIHRoZSBwYXR0ZXJuIHJlY29nbml6ZXIKICAgICAgICBpZiBzZXQoZW50aXRpZXMpICYgKHNldChQYXR0ZXJuUmVjb2duaXplckZhY3RvcnkuUkVDT0dOSVpBQkxFX0VOVElUSUVTLmtleXMoKSkpOgogICAgICAgICAgICBwYXR0ZXJuX3JlY29nbml6ZXJfZmFjdG9yeSA9IFBhdHRlcm5SZWNvZ25pemVyRmFjdG9yeSgpCiAgICAgICAgICAgIGZvciByZWNvZ25pemVyIGluIHBhdHRlcm5fcmVjb2duaXplcl9mYWN0b3J5Ll9jcmVhdGVfcGF0dGVybl9yZWNvZ25pemVyKCk6CiAgICAgICAgICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihyZWNvZ25pemVyKQogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmImFyZ3VtZW50IG9mIG1vZGVsIGFuZCBlbnRpdGllcyBjYW4gbm90IGJlIE5vbmUgYXQgdGhlIHNhbWUgdGltZSIKICAgICAgICApCiAgICBhbmFseXplciA9IHBhLkFuYWx5emVyRW5naW5lKAogICAgICAgIHJlZ2lzdHJ5PXJlZ2lzdHJ5LAogICAgICAgIHN1cHBvcnRlZF9sYW5ndWFnZXM9WyJlbiJdLAogICAgKQoKICAgIHN1cHBvcnRlZF9lbnRpdGllcyA9IGFuYWx5emVyLmdldF9zdXBwb3J0ZWRfZW50aXRpZXMoKQoKICAgIGlmIGVudGl0aWVzIGFuZCBub3QgYWxsKGl0ZW0gaW4gc3VwcG9ydGVkX2VudGl0aWVzIGZvciBpdGVtIGluIGVudGl0aWVzKToKICAgICAgICBub3Rfc3VwcG9ydGVkX2VudGl0aWVzID0gWwogICAgICAgICAgICBpdGVtIGZvciBpdGVtIGluIGVudGl0aWVzIGlmIGl0ZW0gbm90IGluIHN1cHBvcnRlZF9lbnRpdGllcwogICAgICAgIF0KICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlRoZSBjdXJyZW50IG1vZGVsIHttb2RlbH0gZG9lc24ndCBzdXBwb3J0IHRoZSBmb2xsb3dpbmcgZW50aXRpZXM6IHtub3Rfc3VwcG9ydGVkX2VudGl0aWVzfS4gIgogICAgICAgICAgICBmIlN1cHBvcnRlZCBlbnRpdGllcyBhcmU6IHtzdXBwb3J0ZWRfZW50aXRpZXN9IgogICAgICAgICkKICAgIHJldHVybiBhbmFseXplcgoKCmRlZiBfZ2V0X2Fub255bWl6ZXJfZW5naW5lKCkgLT4gcHJlX2Fub3ltaXplci5Bbm9ueW1pemVyRW5naW5lOgogICAgIiIiCiAgICBSZXR1cm4gQW5vbnltaXplckVuZ2luZS4KCiAgICA6cmV0dXJuczogVGhlIEFub255bWl6ZXJFbmdpbmUuCiAgICAiIiIKICAgIHJldHVybiBwcmVfYW5veW1pemVyLkFub255bWl6ZXJFbmdpbmUoKQoKCmRlZiBfYW5vbnltaXplKAogICAgdGV4dDogc3RyLAogICAgYW5hbHl6ZV9yZXN1bHRzOiBMaXN0W3BhLlJlY29nbml6ZXJSZXN1bHRdLAogICAgZW50aXR5X29wZXJhdG9yX21hcDogZGljdCA9IE5vbmUsCiAgICBpc19mdWxsX3RleHQ6IGJvb2wgPSBUcnVlLAopIC0+IHN0cjoKICAgICIiIgogICAgQW5vbnltaXplIGlkZW50aWZpZWQgaW5wdXQgdXNpbmcgUHJlc2lkaW8gQWJvbnltaXplci4KCiAgICA6cGFyYW0gdGV4dDogICAgICAgICAgICAgICAgVGhlIHRleHQgZm9yIGFuYWx5c2lzLgogICAgOnBhcmFtIGFuYWx5emVfcmVzdWx0czogICAgIFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbQogICAgOnBhcmFtIGVudGl0eV9vcGVyYXRvcl9tYXA6IFRoZSBlbnRpdHlfb3BlcmF0b3JfbWFwIGlzIGEgZGljdGlvbmFyeSB0aGF0IG1hcHMgZW50aXR5IHRvIG9wZXJhdG9yIG5hbWUgYW5kIG9wZXJhdG9yIHBhcmFtcy4KICAgIDpwYXJhbSBpc19mdWxsX3RleHQ6ICAgICAgICBXaGV0aGVyIHRoZSB0ZXh0IGlzIGZ1bGwgdGV4dCBvciBub3QuCgogICAgOnJldHVybnM6IFRoZSBhbm9ueW1pemVkIHRleHQuCiAgICAiIiIKICAgIGlmIG5vdCB0ZXh0OgogICAgICAgIHJldHVybiAiIgoKICAgIGFub255bWl6ZXJfZW5naW5lID0gX2dldF9hbm9ueW1pemVyX2VuZ2luZSgpCiAgICBpZiBub3QgZW50aXR5X29wZXJhdG9yX21hcDoKICAgICAgICBvcGVyYXRvcnMgPSBOb25lCiAgICBlbHNlOgogICAgICAgICMgQ3JlYXRlIE9wZXJhdG9yQ29uZmlnIGJhc2VkIG9uIHRoZSBlbnRpdHlfb3BlcmF0b3JfbWFwCiAgICAgICAgb3BlcmF0b3JzID0gewogICAgICAgICAgICBlbnRpdHk6IE9wZXJhdG9yQ29uZmlnKG9wZXJhdG9yX25hbWUsIG9wZXJhdG9yX3BhcmFtcykKICAgICAgICAgICAgZm9yIGVudGl0eSwgKG9wZXJhdG9yX25hbWUsIG9wZXJhdG9yX3BhcmFtcykgaW4gZW50aXR5X29wZXJhdG9yX21hcC5pdGVtcygpCiAgICAgICAgfQoKICAgIGlmIGlzX2Z1bGxfdGV4dDoKICAgICAgICAjIEFub255bWl6ZSB0aGUgZW50aXJlIHRleHQKICAgICAgICByZXR1cm4gYW5vbnltaXplcl9lbmdpbmUuYW5vbnltaXplKAogICAgICAgICAgICB0ZXh0PXRleHQsIGFuYWx5emVyX3Jlc3VsdHM9YW5hbHl6ZV9yZXN1bHRzLCBvcGVyYXRvcnM9b3BlcmF0b3JzCiAgICAgICAgKS50ZXh0CiAgICAjIFRva2VuaXplIHRoZSB0ZXh0IHRvIHNlbnRlbmNlcwogICAgc2VudGVuY2VzID0gbmx0ay5zZW50X3Rva2VuaXplKHRleHQpCiAgICBhbm9ueW1pemVkX3NlbnRlbmNlcyA9IFtdCiAgICBjdXJyZW50X2lkeCA9IDAKCiAgICAjIEZpbmQgdGhlIHNlbnRlbmNlIHRoYXQgaGFzIHBpaSBlbnRpdHkKICAgIGZvciBzZW50ZW5jZSBpbiBzZW50ZW5jZXM6CiAgICAgICAgc3RhcnRfaWR4ID0gY3VycmVudF9pZHgKICAgICAgICBlbmRfaWR4ID0gc3RhcnRfaWR4ICsgbGVuKHNlbnRlbmNlKQoKICAgICAgICAjIEdldCB0aGUgZW50aXRpZXMgdGhhdCBhcmUgaW4gdGhlIHNlbnRlbmNlLCB1cGRhdGUgaHRlIHN0YXJ0X2lkeCBhbmQgZW5kX2lkeAogICAgICAgIHNlbnRlbmNlX3Jlc3VsdHMgPSBbCiAgICAgICAgICAgIHBhLlJlY29nbml6ZXJSZXN1bHQoCiAgICAgICAgICAgICAgICByZXN1bHQuZW50aXR5X3R5cGUsCiAgICAgICAgICAgICAgICBzdGFydD1yZXN1bHQuc3RhcnQgLSBzdGFydF9pZHgsCiAgICAgICAgICAgICAgICBlbmQ9cmVzdWx0LmVuZCAtIHN0YXJ0X2lkeCwKICAgICAgICAgICAgICAgIHNjb3JlPXJlc3VsdC5zY29yZSwKICAgICAgICAgICAgKQogICAgICAgICAgICBmb3IgcmVzdWx0IGluIGFuYWx5emVfcmVzdWx0cwogICAgICAgICAgICBpZiByZXN1bHQuc3RhcnQgPj0gc3RhcnRfaWR4IGFuZCByZXN1bHQuZW5kIDw9IGVuZF9pZHgKICAgICAgICBdCgogICAgICAgICMgSWYgUElJIGlzIGRldGVjdGVkCiAgICAgICAgaWYgc2VudGVuY2VfcmVzdWx0czoKICAgICAgICAgICAgYW5vbnltaXplZF9zZW50ZW5jZSA9IGFub255bWl6ZXJfZW5naW5lLmFub255bWl6ZSgKICAgICAgICAgICAgICAgIHRleHQ9c2VudGVuY2UsIGFuYWx5emVyX3Jlc3VsdHM9c2VudGVuY2VfcmVzdWx0cywgb3BlcmF0b3JzPW9wZXJhdG9ycwogICAgICAgICAgICApLnRleHQKICAgICAgICAgICAgYW5vbnltaXplZF9zZW50ZW5jZXMuYXBwZW5kKGFub255bWl6ZWRfc2VudGVuY2UpCgogICAgICAgIGN1cnJlbnRfaWR4ID0gZW5kX2lkeAoKICAgIHJldHVybiAiICIuam9pbihhbm9ueW1pemVkX3NlbnRlbmNlcykKCgpkZWYgX2dldF90b2tlbnMoCiAgICB0ZXh0OiBzdHIsIGFuYWx5emVfcmVzdWx0czogTGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XSwgaXNfZnVsbDogYm9vbCA9IFRydWUKKSAtPiBMaXN0W3N0cl06CiAgICAiIiIKICAgIEdldCB0aGUgZnVsbCB0b2tlbnMgb3Igb25seSBjb250YWlucyB0aGUgZW50aXRpZXMgdGhhdCBjYW4gZm9ybSBhIHNlbnRlbmNlLgoKICAgIDpwYXJhbSB0ZXh0OiAgICAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgIDpwYXJhbSBhbmFseXplX3Jlc3VsdHM6IFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbQogICAgOnBhcmFtIGlzX2Z1bGw6ICAgICAgICAgV2hldGhlciByZXR1cm4gZnVsbCB0b2tlbnMgb3IganVzdCB0aGUgdG9rZW5zIHRoYXQgb25seSBjb250YWlucyB0aGUgZW50aXRpZXMgdGhhdCBjYW4gZm9ybSBhIHNlbnRlbmNlLgoKICAgIDpyZXR1cm5zOiBUaGUgdG9rZW5zLgogICAgIiIiCgogICAgdG9rZW5zID0gW10KICAgICMgc29ydCBieSBzdGFydCBpbmRleAogICAgcmVzdWx0cyA9IHNvcnRlZChhbmFseXplX3Jlc3VsdHMsIGtleT1sYW1iZGEgeDogeC5zdGFydCkKICAgIGZvciBpLCByZXMgaW4gZW51bWVyYXRlKHJlc3VsdHMpOgogICAgICAgIGlmIGkgPT0gMDoKICAgICAgICAgICAgdG9rZW5zLmFwcGVuZCh0ZXh0WzogcmVzLnN0YXJ0XSkKCiAgICAgICAgIyBhcHBlbmQgZW50aXR5IHRleHQgYW5kIGVudGl0eSB0eXBlCiAgICAgICAgdG9rZW5zLmFwcGVuZCgodGV4dFtyZXMuc3RhcnQgOiByZXMuZW5kXSwgcmVzLmVudGl0eV90eXBlKSkKCiAgICAgICAgIyBpZiBhbm90aGVyIGVudGl0eSBjb21pbmcgaS5lLiB3ZSdyZSBub3QgYXQgdGhlIGxhc3QgcmVzdWx0cyBlbGVtZW50LAogICAgICAgICMgYWRkIHRleHQgdXAgdG8gbmV4dCBlbnRpdHkKICAgICAgICBpZiBpICE9IGxlbihyZXN1bHRzKSAtIDE6CiAgICAgICAgICAgIHRva2Vucy5hcHBlbmQodGV4dFtyZXMuZW5kIDogcmVzdWx0c1tpICsgMV0uc3RhcnRdKQogICAgICAgICMgaWYgbm8gbW9yZSBlbnRpdGllcyBjb21pbmcsIGFkZCBhbGwgcmVtYWluaW5nIHRleHQKICAgICAgICBlbHNlOgogICAgICAgICAgICB0b2tlbnMuYXBwZW5kKHRleHRbcmVzLmVuZCA6XSkKCiAgICAjIGdldCB0aGUgdG9rZW5zIHRoYXQgb25seSBjb250YWlucyB0aGUgZW50aXRpZXMgdGhhdCBjYW4gZm9ybSBhIHNlbnRlbmNlCiAgICBwYXJ0X2Fubm9udGF0ZWRfdG9rZW5zID0gW10KICAgIGlmIG5vdCBpc19mdWxsOgogICAgICAgIGxhc3RfZW5kX3NlbnRlbmNlID0gMAogICAgICAgIGZvciBpLCB0b2tlbiBpbiBlbnVtZXJhdGUodG9rZW5zKToKICAgICAgICAgICAgaWYgYW55KGl0ZW0gaW4gdG9rZW4gZm9yIGl0ZW0gaW4gWyIuIiwgIiEiLCAiPyJdKSBhbmQgYW55KAogICAgICAgICAgICAgICAgdHlwZShpdGVtKSBpcyB0dXBsZSBmb3IgaXRlbSBpbiB0b2tlbnNbbGFzdF9lbmRfc2VudGVuY2U6aV0KICAgICAgICAgICAgKToKICAgICAgICAgICAgICAgIHBhcnRfYW5ub250YXRlZF90b2tlbnMuYXBwZW5kKHRva2Vuc1tsYXN0X2VuZF9zZW50ZW5jZTppXSkKICAgICAgICAgICAgICAgIGxhc3RfZW5kX3NlbnRlbmNlID0gaQogICAgICAgIHJldHVybiBwYXJ0X2Fubm9udGF0ZWRfdG9rZW5zCiAgICByZXR1cm4gdG9rZW5zCgoKZGVmIF9hbm5vdGF0ZSgKICAgIHRleHQ6IHN0ciwgc3RfYW5hbHl6ZV9yZXN1bHRzOiBMaXN0W3BhLlJlY29nbml6ZXJSZXN1bHRdLCBpc19mdWxsX2h0bWw6IGJvb2wgPSBUcnVlCikgLT4gTGlzdFtzdHJdOgogICAgIiIiCiAgICBBbm5vdGF0ZSBpZGVudGlmaWVkIGlucHV0IHVzaW5nIFByZXNpZGlvIEFub255bWl6ZXIuCgogICAgOnBhcmFtIHRleHQ6ICAgICAgICAgICAgICAgVGhlIHRleHQgZm9yIGFuYWx5c2lzLgogICAgOnBhcmFtIHN0X2FuYWx5emVfcmVzdWx0czogVGhlIGxpc3Qgb2YgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBjb25zdHJ1Y3RlZCBmcm9tIGFuYWx5c2lzLgogICAgOnBhcmFtIGlzX2Z1bGxfaHRtbDogICAgICAgV2hldGhlciBnZW5lcmF0ZSBmdWxsIGh0bWwgb3Igbm90LgoKICAgIDpyZXR1cm5zOiBUaGUgbGlzdCBvZiB0b2tlbnMgd2l0aCB0aGUgaWRlbnRpZmllZCBlbnRpdGllcy4KCiAgICAiIiIKICAgIHJldHVybiBfZ2V0X3Rva2Vucyh0ZXh0LCBzdF9hbmFseXplX3Jlc3VsdHMsIGlzX2Z1bGxfaHRtbCkKCgpkZWYgX3Byb2Nlc3MoCiAgICB0ZXh0OiBzdHIsCiAgICBtb2RlbDogcGEuQW5hbHl6ZXJFbmdpbmUsCiAgICBzY29yZV90aHJlc2hvbGQ6IGZsb2F0LAogICAgZW50aXRpZXM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBlbnRpdGllc19vcGVyYXRvcl9tYXA6IGRpY3QgPSBOb25lLAogICAgaXNfZnVsbF90ZXh0OiBib29sID0gVHJ1ZSwKKSAtPiBUdXBsZVtzdHIsIGxpc3RdOgogICAgIiIiCiAgICBQcm9jZXNzIHRoZSB0ZXh0IG9mIHN0ciB1c2luZyB0aGUgbW9kZWwuCgogICAgOnBhcmFtIHRleHQ6ICAgICAgICAgICAgICAgICAgVGV4dCB0byBwcm9jZXNzCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgICAgICAgICBNb2RlbCB0byB1c2UgZm9yIHByb2Nlc3NpbmcKICAgIDpwYXJhbSBlbnRpdGllczogICAgICAgICAgICAgIEVudGl0aWVzIHRvIHJlY29nbml6ZQogICAgOnBhcmFtIGVudGl0aWVzX29wZXJhdG9yX21hcDogVGhlIGVudGl0eV9vcGVyYXRvcl9tYXAgaXMgYSBkaWN0aW9uYXJ5IHRoYXQgbWFwcyBlbnRpdHkgdG8gb3BlcmF0b3IgbmFtZSBhbmQgb3BlcmF0b3IgcGFyYW1zLgogICAgOnBhcmFtIHNjb3JlX3RocmVzaG9sZDogICAgICAgVGhlIHNjb3JlIHRocmVzaG9sZCB0byB1c2UgZm9yIHJlY29nbml0aW9uCiAgICA6cGFyYW0gaXNfZnVsbF90ZXh0OiAgICAgICAgICBXaGV0aGVyIHRvIHJldHVybiB0aGUgZnVsbCB0ZXh0IG9yIGp1c3QgdGhlIGFubm90YXRlZCB0ZXh0CgogICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CgogICAgICAgICAgICAgICogdGhlIGFub255bWl6ZWQgdGV4dAogICAgICAgICAgICAgICogdGhlIGxpc3Qgb2YgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBjb25zdHJ1Y3RlZCBmcm9tIGFuYWx5c2lzCiAgICAiIiIKCiAgICAjIGdldCB0aGUgYW5hbHl6ZXIgZW5naW5lCiAgICBhbmFseXplciA9IG1vZGVsCgogICAgIyBhbmFseXplIHRoZSB0ZXh0IHRoYXQgY2FuIGJlIHVzZWQgZm9yIGFub255bWl6YXRpb24KICAgIHJlc3VsdHMgPSBhbmFseXplci5hbmFseXplKAogICAgICAgIHRleHQ9dGV4dCwKICAgICAgICBsYW5ndWFnZT0iZW4iLAogICAgICAgIGVudGl0aWVzPWVudGl0aWVzLAogICAgICAgIHNjb3JlX3RocmVzaG9sZD1zY29yZV90aHJlc2hvbGQsCiAgICAgICAgcmV0dXJuX2RlY2lzaW9uX3Byb2Nlc3M9VHJ1ZSwKICAgICkKCiAgICAjIGFub255bWl6ZSB0aGUgdGV4dCwgcmVwbGFjZSB0aGUgcGlpIGVudGl0aWVzIHdpdGggdGhlIGxhYmVscwogICAgYW5vbnltaXplZF90ZXh0ID0gX2Fub255bWl6ZSh0ZXh0LCByZXN1bHRzLCBlbnRpdGllc19vcGVyYXRvcl9tYXAsIGlzX2Z1bGxfdGV4dCkKCiAgICByZXR1cm4gYW5vbnltaXplZF90ZXh0LCByZXN1bHRzCgoKZGVmIF9nZXRfc2luZ2xlX2h0bWwoCiAgICB0ZXh0OiBzdHIsIHJlc3VsdHM6IExpc3RbcGEuUmVjb2duaXplclJlc3VsdF0sIGlzX2Z1bGxfaHRtbDogYm9vbCA9IFRydWUKKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIGh0bWwgZm9yIGEgc2luZ2xlIHR4dCBmaWxlLgoKICAgIDpwYXJhbSB0ZXh0OiAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgIDpwYXJhbSByZXN1bHRzOiAgICAgIFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX2h0bWw6IFdoZXRoZXIgZ2VuZXJhdGUgZnVsbCBodG1sIG9yIG5vdC4KCiAgICA6cmV0dXJuczogVGhlIGh0bWwgc3RyaW5nIGZvciBhIHNpbmdsZSB0eHQgZmlsZS4KICAgICIiIgogICAgIyBjb252ZXJ0IHRoZSByZXN1bHRzIHRvIHRva2VucyB0byBnZW5lcmF0ZSB0aGUgaHRtbAogICAgdG9rZW5zID0gX2Fubm90YXRlKHRleHQsIHJlc3VsdHMsIGlzX2Z1bGxfaHRtbCkKICAgIGh0bWwgPSBhdF91dGlsLmdldF9hbm5vdGF0ZWRfaHRtbCgqdG9rZW5zKQoKICAgICMgYXZvaWQgdGhlIGVycm9yIGR1cmluZyByZW5kZXJpbmcgb2YgdGhlIFxuIGluIHRoZSBodG1sCiAgICBiYWNrc2xhc2hfY2hhciA9ICJcXCIKCiAgICBodG1sX3N0ciA9IGYiPHA+e2h0bWwucmVwbGFjZSgne2JhY2tzbGFzaF9jaGFyfW4nLCAnPGJyPicpfTwvcD4iCgogICAgcmV0dXJuIGh0bWxfc3RyCgoKZGVmIF9nZXRfc2luZ2xlX2pzb24ocmVzdWx0czogTGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XSwgaXNfZnVsbF9yZXBvcnQ6IGJvb2wgPSBUcnVlKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIGpzb24gZm9yIGEgc2luZ2xlIHR4dCBmaWxlLgoKICAgIDpwYXJhbSByZXN1bHRzOiAgICAgICAgVGhlIGxpc3Qgb2YgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBjb25zdHJ1Y3RlZCBmcm9tIGFuYWx5c2lzLgogICAgOnBhcmFtIGlzX2Z1bGxfcmVwb3J0OiBXaGV0aGVyIGdlbmVyYXRlIGZ1bGwganNvbiBvciBub3QuCgogICAgOnJldHVybnM6IFRoZSBqc29uIHN0cmluZyBmb3IgYSBzaW5nbGUgdHh0IGZpbGUuCiAgICAiIiIKICAgICMgZ2VuZXJhdGUgdGhlIHN0YXRzIHJlcG9ydCBpZiBuZWVkZWQKICAgIGlmIG5vdCBpc19mdWxsX3JlcG9ydDoKICAgICAgICBzdGF0cyA9IFtdCiAgICAgICAgIyBhZGQgdGhlIHNpbXBsaWZ5IHN0YXRzIGxvZ2ljIGhlcmUKICAgICAgICBmb3IgaXRlbSBpbiByZXN1bHRzOgogICAgICAgICAgICBpdGVtLmFuYWx5c2lzX2V4cGxhbmF0aW9uID0gTm9uZQogICAgICAgICAgICBzdGF0cy5hcHBlbmQoaXRlbSkKICAgIGVsc2U6CiAgICAgICAgc3RhdHMgPSByZXN1bHRzCgogICAgcmV0dXJuIHN0YXRzCgoKZGVmIF9nZXRfYWxsX2h0bWwoCiAgICB0eHRfY29udGVudDogZGljdCwKICAgIHJlc19kaWN0OiBkaWN0LAogICAgaXNfZnVsbF9odG1sOiBib29sID0gVHJ1ZSwKKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIGh0bWwgZm9yIGFsbCB0eHQgZmlsZXMuCgogICAgOnBhcmFtIHR4dF9jb250ZW50OiAgVGhlIGRpY3Rpb25hcnkgb2YgdHh0IGZpbGUgbmFtZSBhbmQgY29udGVudC4KICAgIDpwYXJhbSByZXNfZGljdDogICAgIFRoZSBkaWN0aW9uYXJ5IG9mIHR4dCBmaWxlIG5hbWUgYW5kIHRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX2h0bWw6IFdoZXRoZXIgZ2VuZXJhdGUgZnVsbCBodG1sIG9yIG5vdC4KCiAgICA6cmV0dXJuczogVGhlIGh0bWwgc3RyaW5nIGZvciBhbGwgdHh0IGZpbGVzLgoKICAgICIiIgogICAgIyBUaGVzZSBhcmUgcGxhY2Vob2xkZXIgZm9yIHRoZSBodG1sIHN0cmluZwogICAgaHRtbF9pbmRleCA9ICI8aHRtbD48aGVhZD48dGl0bGU+SGlnaGxpZ2h0ZWQgUGlpIEVudGl0aWVzPC90aXRsZT48L2hlYWQ+PGJvZHk+PGgxPkhpZ2hsaWdodGVkIFBpaSBFbnRpdGllczwvaDE+PHVsPiIKICAgIGh0bWxfY29udGVudCA9ICIiCiAgICBmb3IgdHh0X2ZpbGUsIHJlc3VsdHMgaW4gcmVzX2RpY3QuaXRlbXMoKToKICAgICAgICB0eHQgPSB0eHRfY29udGVudFt0eHRfZmlsZV0KICAgICAgICBodG1sX2luZGV4ICs9IGYiPGxpPjxhIGhyZWY9JyN7dHh0X2ZpbGV9Jz57dHh0X2ZpbGV9PC9hPjwvbGk+IgogICAgICAgIGh0bWxfY29udGVudCArPSBmIjxsaT48aDI+e3R4dF9maWxlfTwvaDI+PHA+e19nZXRfc2luZ2xlX2h0bWwodHh0LCByZXN1bHRzLCBpc19mdWxsX2h0bWwpfTwvcD48L2xpPiIKICAgIGh0bWxfaW5kZXggKz0gIjwvdWw+IgogICAgaHRtbF9yZXMgPSBmIntodG1sX2luZGV4fXtodG1sX2NvbnRlbnR9PC9ib2R5PjwvaHRtbD4iCgogICAgcmV0dXJuIGh0bWxfcmVzCgoKZGVmIF9nZXRfYWxsX3JwdChyZXNfZGljdDogZGljdCwgaXNfZnVsbF9yZXBvcnQ6IGJvb2wgPSBUcnVlKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIHN0YXRzIHJlcG9ydCBmb3IgYWxsIHR4dCBmaWxlcy4KCiAgICA6cGFyYW0gcmVzX2RpY3Q6ICAgICAgIFRoZSBkaWN0aW9uYXJ5IG9mIHR4dCBmaWxlIG5hbWUgYW5kIHRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX3JlcG9ydDogV2hldGhlciBnZW5lcmF0ZSBmdWxsIHJlcG9ydCBvciBub3QuCgogICAgOnJldHVybnM6IFRoZSBzdGF0cyByZXBvcnQgZm9yIGFsbCB0eHQgZmlsZXMuCiAgICAiIiIKICAgICMgVGhlc2UgYXJlIHBsYWNlaG9sZGVyIGZvciB0aGUganNvbiByZXBvcnQKICAgIHN0YXRzX2RpY3QgPSB7fQogICAgZm9yIHR4dF9maWxlLCByZXN1bHRzIGluIHJlc19kaWN0Lml0ZW1zKCk6CiAgICAgICAgbmV3X3N0YXRzID0gW10KICAgICAgICBmb3IgaXRlbSBpbiBfZ2V0X3NpbmdsZV9qc29uKHJlc3VsdHMsIGlzX2Z1bGxfcmVwb3J0KToKICAgICAgICAgICAgaWYgaXNfZnVsbF9yZXBvcnQ6CiAgICAgICAgICAgICAgICBpdGVtLmFuYWx5c2lzX2V4cGxhbmF0aW9uID0gaXRlbS5hbmFseXNpc19leHBsYW5hdGlvbi50b19kaWN0KCkKICAgICAgICAgICAgICAgIG5ld19zdGF0cy5hcHBlbmQoaXRlbS50b19kaWN0KCkpCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICB0bXBfZGljdCA9IGl0ZW0udG9fZGljdCgpCiAgICAgICAgICAgICAgICB0bXBfZGljdC5wb3AoImFuYWx5c2lzX2V4cGxhbmF0aW9uIikKICAgICAgICAgICAgICAgIHRtcF9kaWN0LnBvcCgicmVjb2duaXRpb25fbWV0YWRhdGEiKQogICAgICAgICAgICAgICAgbmV3X3N0YXRzLmFwcGVuZCh0bXBfZGljdCkKICAgICAgICBzdGF0c19kaWN0W3R4dF9maWxlXSA9IG5ld19zdGF0cwogICAgcmV0dXJuIHN0YXRzX2RpY3QKCgpkZWYgcmVjb2duaXplX3BpaSgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgaW5wdXRfcGF0aDogVW5pb25bc3RyLCBwYXRobGliLlBhdGhdLAogICAgaHRtbF9rZXk6IHN0ciwKICAgIHNjb3JlX3RocmVzaG9sZDogZmxvYXQsCiAgICBvdXRwdXRfZGlyZWN0b3J5OiBzdHIgPSBOb25lLAogICAgZW50aXRpZXM6IExpc3RbCiAgICAgICAgc3RyCiAgICBdID0gTm9uZSwgICMgTGlzdCBvZiBlbnRpdGllcyB0byByZWNvZ25pemUsIGRlZmF1bHQgaXMgcmVjb2duaXppbmcgYWxsCiAgICBlbnRpdHlfb3BlcmF0b3JfbWFwOiBkaWN0ID0gTm9uZSwKICAgIG1vZGVsOiBzdHIgPSBOb25lLAogICAgZ2VuZXJhdGVfanNvbjogYm9vbCA9IFRydWUsCiAgICBnZW5lcmF0ZV9odG1sOiBib29sID0gVHJ1ZSwKICAgIGlzX2Z1bGxfdGV4dDogYm9vbCA9IFRydWUsCiAgICBpc19mdWxsX2h0bWw6IGJvb2wgPSBUcnVlLAogICAgaXNfZnVsbF9yZXBvcnQ6IGJvb2wgPSBUcnVlLAopIC0+IFVuaW9uW1R1cGxlW3N0ciwgcGQuRGF0YUZyYW1lLCBkaWN0LCBkaWN0XSwgVHVwbGVbc3RyLCBwZC5EYXRhRnJhbWUsIGRpY3RdXToKICAgICIiIgogICAgV2FsayB0aHJvdWdoIHRoZSBpbnB1dCBwYXRoLCByZWNvZ25pemUgUElJIGluIHRleHQgYW5kIHN0b3JlIHRoZSBhbm9ueW1pemVkIHRleHQgaW4gdGhlIG91dHB1dCBwYXRoLgogICAgR2VuZXJhdGUgdGhlIGh0bWwgd2l0aCBkaWZmZXJlbnQgY29sb3JzIGZvciBlYWNoIGVudGl0eSwganNvbiByZXBvcnQgb2YgdGhlIGV4cGxhbmF0aW9uLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgVGhlIE1MUnVuIGNvbnRleHQuIHRoaXMgaXMgbmVlZGVkIGZvciBsb2cgdGhlIGFydGlmYWN0cy4KICAgIDpwYXJhbSBpbnB1dF9wYXRoOiAgICAgICAgICAgVGhlIGlucHV0IHBhdGggb2YgdGhlIHRleHQgZmlsZXMgbmVlZHMgdG8gYmUgYW5hbHl6ZWQuCiAgICA6cGFyYW0gaHRtbF9rZXk6ICAgICAgICAgICAgIFRoZSBodG1sIGtleSBmb3IgdGhlIGFydGlmYWN0LgogICAgOnBhcmFtIHNjb3JlX3RocmVzaG9sZDogICAgICBUaGUgc2NvcmUgdGhyZXNob2xkIHRvIG1hcmsgdGhlIHJlY29nbml0aW9uIGFzIHRydXN0ZWQuCiAgICA6cGFyYW0gb3V0cHV0X2RpcmVjdG9yeTogICAgIFRoZSBvdXRwdXQgZGlyZWN0b3J5IHBhdGggdG8gc3RvcmUgdGhlIGFub255bWl6ZWQgdGV4dC4KICAgIDpwYXJhbSBlbnRpdGllczogICAgICAgICAgICAgVGhlIGxpc3Qgb2YgZW50aXRpZXMgdG8gcmVjb2duaXplLgogICAgOnBhcmFtIGVudGl0eV9vcGVyYXRvcl9tYXA6ICBUaGUgbWFwIG9mIGVudGl0eSB0byBvcGVyYXRvciAobWFzaywgcmVkYWN0LCByZXBsYWNlLCBrZWVwLCBoYXNoLCBhbmQgaXRzIHBhcmFtcykKICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgVGhlIG1vZGVsIHRvIHVzZS4gQ2FuIGJlICJzcGFjeSIsICJmbGFpciIsICJwYXR0ZXJuIiBvciAid2hvbGUiLgogICAgOnBhcmFtIGdlbmVyYXRlX2pzb246ICAgICAgICBXaGV0aGVyIHRvIGdlbmVyYXRlIHRoZSBqc29uIHJlcG9ydCBvZiB0aGUgZXhwbGFuYXRpb24uCiAgICA6cGFyYW0gZ2VuZXJhdGVfaHRtbDogICAgICAgIFdoZXRoZXIgdG8gZ2VuZXJhdGUgdGhlIGh0bWwgcmVwb3J0IG9mIHRoZSBleHBsYW5hdGlvbi4KICAgIDpwYXJhbSBpc19mdWxsX3RleHQ6ICAgICAgICAgV2hldGhlciB0byByZXR1cm4gdGhlIGZ1bGwgdGV4dCBvciBvbmx5IHRoZSBtYXNrZWQgdGV4dC4KICAgIDpwYXJhbSBpc19mdWxsX2h0bWw6ICAgICAgICAgV2hldGhlciB0byByZXR1cm4gdGhlIGZ1bGwgaHRtbCBvciBqdXN0IHRoZSBhbm5vdGF0ZWQgdGV4dAogICAgOnBhcmFtIGlzX2Z1bGxfcmVwb3J0OiAgICAgICBXaGV0aGVyIHRvIHJldHVybiB0aGUgZnVsbCByZXBvcnQgb3IganVzdCB0aGUgc2NvcmUgYW5kIHN0YXJ0LCBlbmQgaW5kZXgKCiAgICA6cmV0dXJuczogQSB0dXBsZSBvZjoKCiAgICAgICAgICAgICAgKiBQYXRoIHRvIHRoZSBvdXRwdXQgZGlyZWN0b3J5CiAgICAgICAgICAgICAgKiBUaGUganNvbiByZXBvcnQgb2YgdGhlIGV4cGxhbmF0aW9uIChpZiBnZW5lcmF0ZV9qc29uIGlzIFRydWUpCiAgICAgICAgICAgICAgKiBBIGRpY3Rpb25hcnkgb2YgZXJyb3JzIGZpbGVzIHRoYXQgd2VyZSBub3QgcHJvY2Vzc2VkCgogICAgIiIiCgogICAgIyBTZXQgb3V0cHV0IGRpcmVjdG9yeQogICAgaWYgb3V0cHV0X2RpcmVjdG9yeSBpcyBOb25lOgogICAgICAgIG91dHB1dF9kaXJlY3RvcnkgPSB0ZW1wZmlsZS5ta2R0ZW1wKCkKCiAgICAjIENyZWF0ZSB0aGUgb3V0cHV0IGRpcmVjdG9yeToKICAgIG91dHB1dF9kaXJlY3RvcnkgPSBwYXRobGliLlBhdGgob3V0cHV0X2RpcmVjdG9yeSkKICAgIGlmIG5vdCBvdXRwdXRfZGlyZWN0b3J5LmV4aXN0cygpOgogICAgICAgIG91dHB1dF9kaXJlY3RvcnkubWtkaXIocGFyZW50cz1UcnVlLCBleGlzdF9vaz1UcnVlKQoKICAgIHR4dF9maWxlc19kaXJlY3RvcnkgPSBwYXRobGliLlBhdGgoaW5wdXRfcGF0aCkKICAgIHN1Y2Nlc3NlcyA9IFtdCiAgICBlcnJvcnMgPSB7fQoKICAgIHJlc19kaWN0ID0ge30KICAgIHR4dF9jb250ZW50ID0ge30KICAgICMgTG9hZCB0aGUgbW9kZWw6CiAgICBhbmFseXplciA9IF9nZXRfYW5hbHl6ZXJfZW5naW5lKG1vZGVsLCBlbnRpdGllcykKICAgIGxvZ2dlci5pbmZvKCJNb2RlbCBsb2FkZWQiKQogICAgIyBHbyBvdmVyIHRoZSB0ZXh0IGZpbGVzIGluIHRoZSBpbnB1dCBwYXRoLCBhbmFseXplIGFuZCBhbm9ueW1pemUgdGhlbToKICAgIGZvciB0eHRfZmlsZSBpbiB0cWRtKAogICAgICAgIGxpc3QodHh0X2ZpbGVzX2RpcmVjdG9yeS5nbG9iKCIqLnR4dCIpKSwKICAgICAgICBkZXNjPSJQcm9jZXNzaW5nIGZpbGVzIiwKICAgICAgICB1bml0PSJmaWxlIiwKICAgICk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICAjIExvYWQgdGhlIHN0ciBmcm9tIHRoZSB0ZXh0IGZpbGUKICAgICAgICAgICAgdGV4dCA9IHR4dF9maWxlLnJlYWRfdGV4dCgpCiAgICAgICAgICAgIHR4dF9jb250ZW50W3N0cih0eHRfZmlsZSldID0gdGV4dAogICAgICAgICAgICAjIFByb2Nlc3MgdGhlIHRleHQgdG8gcmVjb2dpbnplIHRoZSBwaWkgZW50aXRpZXMgaW4gaXQKICAgICAgICAgICAgYW5vbnltaXplZF90ZXh0LCByZXN1bHRzID0gX3Byb2Nlc3MoCiAgICAgICAgICAgICAgICB0ZXh0PXRleHQsCiAgICAgICAgICAgICAgICBtb2RlbD1hbmFseXplciwKICAgICAgICAgICAgICAgIGVudGl0aWVzPWVudGl0aWVzLAogICAgICAgICAgICAgICAgZW50aXRpZXNfb3BlcmF0b3JfbWFwPWVudGl0eV9vcGVyYXRvcl9tYXAsCiAgICAgICAgICAgICAgICBzY29yZV90aHJlc2hvbGQ9c2NvcmVfdGhyZXNob2xkLAogICAgICAgICAgICAgICAgaXNfZnVsbF90ZXh0PWlzX2Z1bGxfdGV4dCwKICAgICAgICAgICAgKQogICAgICAgICAgICByZXNfZGljdFtzdHIodHh0X2ZpbGUpXSA9IHJlc3VsdHMKICAgICAgICAgICAgIyBTdG9yZSB0aGUgYW5vbnltaXplZCB0ZXh0IGluIHRoZSBvdXRwdXQgcGF0aAogICAgICAgICAgICBvdXRwdXRfZmlsZSA9IG91dHB1dF9kaXJlY3RvcnkgLyBmInt0eHRfZmlsZS5zdGVtfS50eHQiCiAgICAgICAgICAgIG91dHB1dF9maWxlLnBhcmVudC5ta2RpcihwYXJlbnRzPVRydWUsIGV4aXN0X29rPVRydWUpCiAgICAgICAgICAgIHdpdGggb3BlbihvdXRwdXRfZmlsZSwgInciKSBhcyBmOgogICAgICAgICAgICAgICAgZi53cml0ZShhbm9ueW1pemVkX3RleHQpCiAgICAgICAgICAgIHN1Y2Nlc3Nlcy5hcHBlbmQoW3R4dF9maWxlLm5hbWUsIG91dHB1dF9maWxlLm5hbWVdKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICAgICAgZXJyb3JzW3N0cih0eHRfZmlsZSldID0gc3RyKGUpCiAgICAgICAgICAgIGxvZ2dlci5lcnJvcihmIkVycm9yIHByb2Nlc3Npbmcge3R4dF9maWxlfToge2V9IikKCiAgICBzdWNjZXNzZXMgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgc3VjY2Vzc2VzLAogICAgICAgIGNvbHVtbnM9WyJvcmlnaW5hbF9maWxlIiwgImFub255bWl6ZWRfZmlsZSJdLAogICAgKQoKICAgIGlmIGdlbmVyYXRlX2h0bWw6CiAgICAgICAgIyBHZW5lcmF0ZSB0aGUgaHRtbCByZXBvcnQKICAgICAgICBodG1sX3JlcyA9IF9nZXRfYWxsX2h0bWwodHh0X2NvbnRlbnQsIHJlc19kaWN0LCBpc19mdWxsX2h0bWwpCiAgICAgICAgIyBTdG9yZSB0aGUgaHRtbCByZXBvcnQgaW4gdGhlIGNvbnRleHQKICAgICAgICBhcnRpX2h0bWwgPSBtbHJ1bi5hcnRpZmFjdHMuQXJ0aWZhY3QoYm9keT1odG1sX3JlcywgZm9ybWF0PSJodG1sIiwga2V5PWh0bWxfa2V5KQogICAgICAgIGNvbnRleHQubG9nX2FydGlmYWN0KGFydGlfaHRtbCkKICAgIGlmIGdlbmVyYXRlX2pzb246CiAgICAgICAgIyBHZW5lcmF0ZSB0aGUganNvbiByZXBvcnQKICAgICAgICBqc29uX3JlcyA9IF9nZXRfYWxsX3JwdChyZXNfZGljdCwgaXNfZnVsbF9yZXBvcnQpCiAgICAgICAgcmV0dXJuIHN0cihvdXRwdXRfZGlyZWN0b3J5KSwgc3VjY2Vzc2VzLCBlcnJvcnMsIGpzb25fcmVzCiAgICByZXR1cm4gc3RyKG91dHB1dF9kaXJlY3RvcnkpLCBzdWNjZXNzZXMsIGVycm9ycwo= base_image: mlrun/mlrun - commands: - - python -m pip install nltk pandas presidio-anonymizer presidio-analyzer torch - flair@git+https://github.com/flairNLP/flair.git@d4ed67bf663e4066517f00397412510d90043653 - st-annotated-text https://huggingface.co/beki/en_spacy_pii_distilbert/resolve/main/en_spacy_pii_distilbert-any-py3-none-any.whl - code_origin: git@github.com-personal:pengwei715/functions.git#5468a7acb9b9fde12832e27daac2624f43746ee7:/Users/Peng_Wei/work/mlrun_related/functions/pii_recognizer/pii_recognizer.py - origin_filename: /Users/Peng_Wei/work/mlrun_related/functions/pii_recognizer/pii_recognizer.py - requirements: [] + commands: [] + code_origin: '' + origin_filename: '' + requirements: + - nltk + - pandas + - presidio-anonymizer + - presidio-analyzer + - torch + - flair@git+https://github.com/flairNLP/flair.git@d4ed67bf663e4066517f00397412510d90043653 + - st-annotated-text + - https://huggingface.co/beki/en_spacy_pii_distilbert/resolve/main/en_spacy_pii_distilbert-any-py3-none-any.whl entry_points: + analyze: + name: analyze + doc: Analyze text and return the results. + parameters: + - name: self + - name: text + type: str + doc: The text for analysis. + - name: entities + type: List[str] + doc: The list of entities to recognize. + - name: nlp_artifacts + type: pa.nlp_engine.NlpArtifacts + doc: Not used by this recognizer but needed for the interface. + default: null + outputs: + - doc: The list of Presidio RecognizerResult constructed from the recognized + Flair detections. + type: List[pa.RecognizerResult] + lineno: 381 + has_varargs: false + has_kwargs: false recognize_pii: name: recognize_pii - doc: Walk through the input path, recognize PII in text and store the anonymized - text in the output path. Generate the html with different colors for each - entity, json report of the explaination. + doc: 'Walk through the input path, recognize PII in text and store the anonymized + text in the output path. + + Generate the html with different colors for each entity, json report of the + explanation.' parameters: - name: context type: MLClientCtx doc: The MLRun context. this is needed for log the artifacts. - default: '' - name: input_path - type: str - doc: The input path of the text files needs to be analyzied. - default: '' - - name: output_path - type: str - doc: The output path to store the anonymized text. - default: '' - - name: output_suffix - type: str - doc: The surfix of output key for the anonymized text. for example if the - input file is pii.txt, the output key is anoymized, the output file name - will be pii_anonymized.txt. - default: '' + type: Union[str, Path] + doc: The input path of the text files needs to be analyzed. - name: html_key type: str doc: The html key for the artifact. - default: '' - name: score_threshold type: float doc: The score threshold to mark the recognition as trusted. - default: '' + - name: output_directory + type: str + doc: The output directory path to store the anonymized text. + default: null - name: entities type: List[str] doc: The list of entities to recognize. @@ -71,11 +90,11 @@ spec: default: null - name: generate_json type: bool - doc: Whether to generate the json report of the explaination. + doc: Whether to generate the json report of the explanation. default: true - name: generate_html type: bool - doc: Whether to generate the html report of the explaination. + doc: Whether to generate the html report of the explanation. default: true - name: is_full_text type: bool @@ -90,40 +109,20 @@ spec: doc: Whether to return the full report or just the score and start, end index default: true outputs: - - default: '' - doc: 'A tuple of:' - lineno: 850 + - doc: 'A tuple of:' + type: Union[Tuple[str, pd.DataFrame, dict, dict], Tuple[str, pd.DataFrame, + dict]] + lineno: 845 + has_varargs: false + has_kwargs: false description: This function is used to recognize PII in a directory of text files default_handler: recognize_pii disable_auto_mount: false clone_target_dir: '' env: [] - resources: - requests: - memory: 1Mi - cpu: 25m - limits: - memory: 20Gi - cpu: '2' priority_class_name: '' preemption_mode: prevent - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: app.iguazio.com/lifecycle - operator: NotIn - values: - - preemptible - - key: eks.amazonaws.com/capacityType - operator: NotIn - values: - - SPOT - - key: node-lifecycle - operator: NotIn - values: - - spot + affinity: null tolerations: null security_context: {} -verbose: false \ No newline at end of file +verbose: false diff --git a/pii_recognizer/item.yaml b/pii_recognizer/item.yaml index 5fa9f0ae4..2f618febc 100644 --- a/pii_recognizer/item.yaml +++ b/pii_recognizer/item.yaml @@ -30,5 +30,5 @@ spec: - st-annotated-text - https://huggingface.co/beki/en_spacy_pii_distilbert/resolve/main/en_spacy_pii_distilbert-any-py3-none-any.whl url: '' -version: 0.1.0 +version: 0.2.0 test_valid: False diff --git a/pii_recognizer/pii_recognizer.py b/pii_recognizer/pii_recognizer.py index 38c0e0ec3..0acc55dcb 100644 --- a/pii_recognizer/pii_recognizer.py +++ b/pii_recognizer/pii_recognizer.py @@ -1,35 +1,32 @@ -# Copyright 2019 Iguazio +# Copyright 2023 Iguazio # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# import logging import os import pathlib import tempfile import warnings -import pandas as pd -from collections.abc import Iterable -from multiprocessing import Pool, cpu_count -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import List, Set, Tuple, Union import annotated_text.util as at_util import mlrun import nltk +import pandas as pd import presidio_analyzer as pa import presidio_anonymizer as pre_anoymizer from presidio_anonymizer.entities import OperatorConfig -from tqdm.auto import tqdm +from tqdm import tqdm try: import flair as fl @@ -393,7 +390,6 @@ def analyze( :param text: The text for analysis. :param entities: The list of entities to recognize. :param nlp_artifacts: Not used by this recognizer but needed for the interface. - :param language: Text language. Supported languages in MODEL_LANGUAGES :returns: The list of Presidio RecognizerResult constructed from the recognized Flair detections. """ @@ -711,11 +707,11 @@ def _process( entities: List[str] = None, entities_operator_map: dict = None, is_full_text: bool = True, -) -> Tuple[str, str, str]: +) -> Tuple[str, list]: """ Process the text of str using the model. - :param txt: Text to process + :param text: Text to process :param model: Model to use for processing :param entities: Entities to recognize :param entities_operator_map: The entity_operator_map is a dictionary that maps entity to operator name and operator params. @@ -729,7 +725,6 @@ def _process( """ # get the analyzer engine - analyzer = model # analyze the text that can be used for anonymization @@ -850,9 +845,9 @@ def _get_all_rpt(res_dict: dict, is_full_report: bool = True): def recognize_pii( context: mlrun.MLClientCtx, input_path: Union[str, pathlib.Path], - output_path: str, html_key: str, score_threshold: float, + output_directory: str = None, entities: List[ str ] = None, # List of entities to recognize, default is recognizing all @@ -863,20 +858,21 @@ def recognize_pii( is_full_text: bool = True, is_full_html: bool = True, is_full_report: bool = True, -) -> Tuple[pathlib.Path, dict, dict]: +) -> Union[Tuple[str, pd.DataFrame, dict, dict], Tuple[str, pd.DataFrame, dict]]: """ - Walk through the input path, recognize PII in text and store the anonymized text in the output path. Generate the html with different colors for each entity, json report of the explaination. + Walk through the input path, recognize PII in text and store the anonymized text in the output path. + Generate the html with different colors for each entity, json report of the explanation. :param context: The MLRun context. this is needed for log the artifacts. - :param input_path: The input path of the text files needs to be analyzied. - :param output_path: The output path to store the anonymized text. + :param input_path: The input path of the text files needs to be analyzed. :param html_key: The html key for the artifact. :param score_threshold: The score threshold to mark the recognition as trusted. + :param output_directory: The output directory path to store the anonymized text. :param entities: The list of entities to recognize. :param entity_operator_map: The map of entity to operator (mask, redact, replace, keep, hash, and its params) :param model: The model to use. Can be "spacy", "flair", "pattern" or "whole". - :param generate_json: Whether to generate the json report of the explaination. - :param generate_html: Whether to generate the html report of the explaination. + :param generate_json: Whether to generate the json report of the explanation. + :param generate_html: Whether to generate the html report of the explanation. :param is_full_text: Whether to return the full text or only the masked text. :param is_full_html: Whether to return the full html or just the annotated text :param is_full_report: Whether to return the full report or just the score and start, end index @@ -884,46 +880,38 @@ def recognize_pii( :returns: A tuple of: * Path to the output directory - * The json report of the explaination (if generate_json is True) + * The json report of the explanation (if generate_json is True) * A dictionary of errors files that were not processed """ # Set output directory - if output_path is None: - output_path = tempfile.mkdtemp() + if output_directory is None: + output_directory = tempfile.mkdtemp() # Create the output directory: - output_directory = pathlib.Path(output_path) + output_directory = pathlib.Path(output_directory) if not output_directory.exists(): - output_directory.mkdir() + output_directory.mkdir(parents=True, exist_ok=True) txt_files_directory = pathlib.Path(input_path) + successes = [] errors = {} res_dict = {} txt_content = {} # Load the model: - try: - analyzer = _get_analyzer_engine(model, entities) - except Exception as e: - errors["model"] = str(e) - logger.error(f"Error when get the model: {e}") - + analyzer = _get_analyzer_engine(model, entities) logger.info("Model loaded") # Go over the text files in the input path, analyze and anonymize them: - for i, txt_file in enumerate( - tqdm( - list(txt_files_directory.glob("*.txt")), - desc="Processing files", - unit="file", - ) + for txt_file in tqdm( + list(txt_files_directory.glob("*.txt")), + desc="Processing files", + unit="file", ): try: # Load the str from the text file text = txt_file.read_text() - # TODO maybe the encoding issue if from this function call of tqdm.read_text() - # Need to fix it later txt_content[str(txt_file)] = text # Process the text to recoginze the pii entities in it anonymized_text, results = _process( @@ -936,158 +924,19 @@ def recognize_pii( ) res_dict[str(txt_file)] = results # Store the anonymized text in the output path - output_file = ( - output_directory - / f"{str(txt_file.relative_to(txt_files_directory)).split('.')[0]}.txt" - ) + output_file = output_directory / f"{txt_file.stem}.txt" output_file.parent.mkdir(parents=True, exist_ok=True) with open(output_file, "w") as f: f.write(anonymized_text) - + successes.append([txt_file.name, output_file.name]) except Exception as e: errors[str(txt_file)] = str(e) logger.error(f"Error processing {txt_file}: {e}") - if generate_html: - # Generate the html report - html_res = _get_all_html(txt_content, res_dict, is_full_html) - # Store the html report in the context - arti_html = mlrun.artifacts.Artifact(body=html_res, format="html", key=html_key) - context.log_artifact(arti_html) - if generate_json: - # Generate the json report - json_res = _get_all_rpt(res_dict, is_full_report) - return output_path, json_res, errors - return output_path, errors - - -def _recognize_pii_one_file( - input_file: str, - output_file: str, - score_threshold: float, - entities: List[ - str - ] = None, # List of entities to recognize, default is recognizing all - entity_operator_map: dict = None, - model: str = None, - is_full_text: bool = True, -) -> Tuple[dict, dict, dict]: - """ - Recognize PII in text and store the anonymized text in the output path. Generate the html with different colors for each entity, json report of the explaination. - :param input_file: The input path of the text files needs to be analyzied. - :param output_file: The output path to store the anonymized text. - :param score_threshold: The score threshold to mark the recognition as trusted. - :param entities: The list of entities to recognize. - :param entity_operator_map: The map of entity to operator (mask, redact, replace, keep, hash, and its params) - :param model: The model to use. Can be "spacy", "flair", "pattern" or "whole". - :param is_full_text: Whether to return the full text or only the masked text. - - :returns: A tuple of: - * A dictionary of the text content of the input file - * A dictionary of the results of the explaination - * A dictionary of errors files that were not processed - """ - errors = {} - res_dict = {} - txt_content = {} - # Load the model: - try: - analyzer = _get_analyzer_engine(model, entities) - except Exception as e: - errors["model"] = str(e) - logger.error(f"Error when get the model: {e}") - - logger.info("Model loaded") - try: - # Load the str from the text file - with open(input_file, "r", encoding="utf-8") as file: - text = file.read() - txt_content[str(input_file)] = text - # Process the text to recoginze the pii entities in it - anonymized_text, results = _process( - text=text, - model=analyzer, - entities=entities, - entities_operator_map=entity_operator_map, - score_threshold=score_threshold, - is_full_text=is_full_text, - ) - res_dict[str(input_file)] = results - with open(output_file, "w", encoding="utf-8") as f: - f.write(anonymized_text) - - except Exception as e: - errors[str(txt_file)] = str(e) - logger.error(f"Error processing {txt_file}: {e}") - - return res_dict, txt_content, errors - - -def recognize_pii_parallel( - context: mlrun.MLClientCtx, - config_input_output: str, - score_threshold: float, - html_key: str, - entities: List[str] = None, - entity_operator_map: Dict = None, - model: str = None, - generate_html: bool = True, - generate_json: bool = True, - is_full_html: bool = True, - is_full_text: bool = True, - is_full_report: bool = True, - num_processes: int = None, -) -> Tuple[dict, dict]: - """Doing a fan-in and fan-out pattern using mutiple processes for cpu node, Since our model is mixed with rule_based and NLP model based. Both Spacy and Flair do not support the cuda GPU natively. For now, we can use all the cores that a CPU offers. - :param context: The MLRun context. this is needed - :param config_input_output csv file which have the input file path and output file path - :param score_threshold: The threshold of the score to recognize the entities - :param html_key: The key of the html report in the context - :entities List of entities to recognize, default is recognizing all - :entity_operator_map The map of the entities and the operator to use. For example, {"PERSON": "replace", "LOCATION": "mask"} - :param model The model to use. Can be "spacy", "flair", "pattern" or "whole". - :param generate_html: Whether to generate the html report - :param generate_json: Whether to generate the json report - :param is_full_html: Whether to generate the full html report - :param is_full_text: Whether to generate the full text in the html report - :param is_full_report: Whether to generate the full json report - :param num_process The number of process to run in parallel - - :returns: A tuple of: - * A json report of the result explaination - * A dictionary of errors files that were not processed - - """ - if num_processes is None: - num_processes = cpu_count() - - # Read the CSV into a DataFrame - config_df = pd.read_csv(config_input_output) - - # Convert DataFrame rows into a list of tuples, each tuple is arguments for `_recognize_pii_one_file` - tasks = [ - ( - row["input_file"], - row["output_file"], - score_threshold, - entities, - entity_operator_map, - model, - is_full_text, - ) - for _, row in config_df.iterrows() - ] - # Create a pool of processes and distribute the tasks - with Pool(processes=num_processes) as pool: - res = pool.starmap(_recognize_pii_one_file, tasks) - # Get the results - res_dict = {} - txt_content = {} - errors = {} - for r in res: - res_dict.update(r[0]) - txt_content.update(r[1]) - errors.update(r[2]) + successes = pd.DataFrame( + successes, + columns=["original_file", "anonymized_file"], + ) if generate_html: # Generate the html report @@ -1098,5 +947,5 @@ def recognize_pii_parallel( if generate_json: # Generate the json report json_res = _get_all_rpt(res_dict, is_full_report) - return json_res, errors - return errors + return str(output_directory), successes, errors, json_res + return str(output_directory), successes, errors diff --git a/speech_diarization/assets/test_data.wav b/pyannote_audio/assets/test_data.wav similarity index 100% rename from speech_diarization/assets/test_data.wav rename to pyannote_audio/assets/test_data.wav diff --git a/pyannote_audio/function.yaml b/pyannote_audio/function.yaml new file mode 100644 index 000000000..1229e0f32 --- /dev/null +++ b/pyannote_audio/function.yaml @@ -0,0 +1,146 @@ +kind: job +metadata: + name: pyannote-audio + tag: '' + hash: 335752327ddd14b62222bd45faa3a88704505b66 + project: '' + labels: + author: guyl + categories: + - Deep Learning + - Huggingface + - Audio +spec: + command: '' + args: [] + image: '' + build: + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGhlYXBxCmltcG9ydCBsb2dnaW5nCmltcG9ydCBvcGVyYXRvcgppbXBvcnQgb3MKaW1wb3J0IHBhdGhsaWIKZnJvbSBmdW5jdG9vbHMgaW1wb3J0IHJlZHVjZSwgd3JhcHMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdCwgTGlzdCwgVHVwbGUsIFVuaW9uCgppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBweWFubm90ZS5hdWRpbwppbXBvcnQgcHlhbm5vdGUuY29yZQppbXBvcnQgdG9yY2gKaW1wb3J0IHRvcmNoYXVkaW8KZnJvbSB0cWRtIGltcG9ydCB0cWRtCgojIEdldCB0aGUgZ2xvYmFsIGxvZ2dlcjoKX0xPR0dFUiA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCkKCgpkZWYgX2NoZWNrX21scnVuX2FuZF9vcGVuX21waSgpIC0+IFR1cGxlWyJtbHJ1bi5NTENsaWVudEN0eCIsICJtcGk0cHkuTVBJLkludHJhY29tbSJdOgogICAgaXNfbXBpID0gRmFsc2UKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KCiAgICAgICAgY29udGV4dCA9IG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KG5hbWU9Im1scnVuIikKICAgICAgICBpc19tcGkgPSBjb250ZXh0LmxhYmVscy5nZXQoImtpbmQiLCAiam9iIikgPT0gIm1waWpvYiIKCiAgICAgICAgaWYgaXNfbXBpOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBmcm9tIG1waTRweSBpbXBvcnQgTVBJCgogICAgICAgICAgICAgICAgcmV0dXJuIGNvbnRleHQsIE1QSS5DT01NX1dPUkxECiAgICAgICAgICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yIGFzIG1waTRweV9ub3RfZm91bmQ6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgICAgICAgICAiVG8gZGlzdHJpYnV0ZSB0aGUgZnVuY3Rpb24gdXNpbmcgTUxSdW4ncyAnbXBpam9iJyB5b3UgbmVlZCB0byBoYXZlIGBtcGk0cHlgIHBhY2thZ2UgaW4geW91ciAiCiAgICAgICAgICAgICAgICAgICAgImludGVycHJldGVyLiBQbGVhc2UgcnVuIGBwaXAgaW5zdGFsbCBtcGk0cHlgIGFuZCBtYWtlIHN1cmUgeW91IGhhdmUgb3Blbi1tcGkuIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgcmFpc2UgbXBpNHB5X25vdF9mb3VuZAogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJldHVybiBjb250ZXh0LCBOb25lCiAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtb2R1bGVfbm90X2ZvdW5kOgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgcmFpc2UgbW9kdWxlX25vdF9mb3VuZAogICAgcmV0dXJuIE5vbmUsIE5vbmUKCgpkZWYgb3Blbl9tcGlfaGFuZGxlcigKICAgIHdvcmtlcl9pbnB1dHM6IExpc3Rbc3RyXSwgcm9vdF93b3JrZXJfaW5wdXRzOiBEaWN0W3N0ciwgQW55XSA9IE5vbmUKKToKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBDaGVjayBmb3IgTUxSdW4gYW5kIE9wZW5NUEkgYXZhaWxhYmlsaXR5OgogICAgY29udGV4dCwgY29tbSA9IF9jaGVja19tbHJ1bl9hbmRfb3Blbl9tcGkoKQoKICAgICMgQ2hlY2sgaWYgTUxSdW4gaXMgYXZhaWxhYmxlLCBzZXQgdGhlIGdsb2JhbCBsb2dnZXIgdG8gTUxSdW4nczoKICAgIGlmIGNvbnRleHQ6CiAgICAgICAgX0xPR0dFUiA9IGNvbnRleHQubG9nZ2VyCgogICAgZGVmIGRlY29yYXRvcihoYW5kbGVyKToKICAgICAgICBpZiBjb21tIGlzIE5vbmUgb3IgY29tbS5HZXRfc2l6ZSgpID09IDE6CiAgICAgICAgICAgIHJldHVybiBoYW5kbGVyCgogICAgICAgIEB3cmFwcyhoYW5kbGVyKQogICAgICAgIGRlZiB3cmFwcGVyKCoqa3dhcmdzKToKICAgICAgICAgICAgIyBHZXQgdGhlIG9wZW4gbXBpIGVudmlyb25tZW50IHByb3BlcnRpZXM6CiAgICAgICAgICAgIHNpemUgPSBjb21tLkdldF9zaXplKCkKICAgICAgICAgICAgcmFuayA9IGNvbW0uR2V0X3JhbmsoKQoKICAgICAgICAgICAgIyBHaXZlIHRoZSBjb3JyZWN0IGNodW5rIG9mIHRoZSB3b3JrZXJzIGlucHV0czoKICAgICAgICAgICAgZm9yIHdvcmtlcl9pbnB1dCBpbiB3b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBrd2FyZ3Nbd29ya2VyX2lucHV0XQogICAgICAgICAgICAgICAgaWYgaW5wdXRfYXJndW1lbnQgaXMgTm9uZToKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgc3RyKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IF9nZXRfYXVkaW9fZmlsZXMoCiAgICAgICAgICAgICAgICAgICAgICAgIGRhdGFfcGF0aD1wYXRobGliLlBhdGgoaW5wdXRfYXJndW1lbnQpLmFic29sdXRlKCkKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBpZiBsZW4oaW5wdXRfYXJndW1lbnQpIDwgc2l6ZToKICAgICAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgICAgICAgICBmIkNhbm5vdCBzcGxpdCB0aGUgaW5wdXQgJ3t3b3JrZXJfaW5wdXR9JyBvZiBsZW5ndGgge2xlbihpbnB1dF9hcmd1bWVudCl9IHRvIHtzaXplfSB3b3JrZXJzLiAiCiAgICAgICAgICAgICAgICAgICAgICAgIGYiUGxlYXNlIHJlZHVjZSB0aGUgYW1vdW50IG9mIHdvcmtlcnMgZm9yIHRoaXMgaW5wdXQuIgogICAgICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGV2ZW5fY2h1bmtfc2l6ZSA9IGxlbihpbnB1dF9hcmd1bWVudCkgLy8gc2l6ZQogICAgICAgICAgICAgICAgY2h1bmtfc3RhcnQgPSByYW5rICogZXZlbl9jaHVua19zaXplCiAgICAgICAgICAgICAgICBjaHVua19lbmQgPSAoCiAgICAgICAgICAgICAgICAgICAgKHJhbmsgKyAxKSAqIGV2ZW5fY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgICAgIGlmIHJhbmsgKyAxIDwgc2l6ZQogICAgICAgICAgICAgICAgICAgIGVsc2UgbGVuKGlucHV0X2FyZ3VtZW50KQogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICBmIlJhbmsgI3tyYW5rfTogUHJvY2Vzc2luZyBpbnB1dCBjaHVuayBvZiAne3dvcmtlcl9pbnB1dH0nICIKICAgICAgICAgICAgICAgICAgICBmImZyb20gaW5kZXgge2NodW5rX3N0YXJ0fSB0byB7Y2h1bmtfZW5kfS4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBpZiBpc2luc3RhbmNlKGlucHV0X2FyZ3VtZW50LCBsaXN0KToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IGlucHV0X2FyZ3VtZW50W2NodW5rX3N0YXJ0OmNodW5rX2VuZF0KICAgICAgICAgICAgICAgIGVsaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgcGQuRGF0YUZyYW1lKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IGlucHV0X2FyZ3VtZW50Lmlsb2NbY2h1bmtfc3RhcnQ6Y2h1bmtfZW5kOiwgOl0KICAgICAgICAgICAgICAgIGt3YXJnc1t3b3JrZXJfaW5wdXRdID0gaW5wdXRfYXJndW1lbnQKCiAgICAgICAgICAgICMgU2V0IHRoZSByb290IHdvcmtlciBvbmx5IGFyZ3VtZW50czoKICAgICAgICAgICAgaWYgcmFuayA9PSAwIGFuZCByb290X3dvcmtlcl9pbnB1dHM6CiAgICAgICAgICAgICAgICBrd2FyZ3MudXBkYXRlKHJvb3Rfd29ya2VyX2lucHV0cykKCiAgICAgICAgICAgICMgUnVuIHRoZSB3b3JrZXI6CiAgICAgICAgICAgIG91dHB1dCA9IGhhbmRsZXIoKiprd2FyZ3MpCgogICAgICAgICAgICAjIFNlbmQgdGhlIG91dHB1dCB0byB0aGUgcm9vdCByYW5rIChyYW5rICMwKToKICAgICAgICAgICAgb3V0cHV0ID0gY29tbS5nYXRoZXIob3V0cHV0LCByb290PTApCiAgICAgICAgICAgIGlmIHJhbmsgPT0gMDoKICAgICAgICAgICAgICAgICMgSm9pbiB0aGUgb3V0cHV0czoKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkNvbGxlY3RpbmcgZGF0YSBmcm9tIHdvcmtlcnMgdG8gcm9vdCB3b3JrZXIuIikKICAgICAgICAgICAgICAgIGRpYXJpemF0aW9uX2RpY3Rpb25hcnkgPSByZWR1Y2UoCiAgICAgICAgICAgICAgICAgICAgb3BlcmF0b3IuaW9yLCBbZGlhIGZvciBkaWEsIF8gaW4gb3V0cHV0XSwge30KICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGVycm9yc19kaWN0aW9uYXJ5ID0gcmVkdWNlKG9wZXJhdG9yLmlvciwgW2VyciBmb3IgXywgZXJyIGluIG91dHB1dF0sIHt9KQogICAgICAgICAgICAgICAgcmV0dXJuIGRpYXJpemF0aW9uX2RpY3Rpb25hcnksIGVycm9yc19kaWN0aW9uYXJ5CiAgICAgICAgICAgIHJldHVybiBOb25lCgogICAgICAgIHJldHVybiB3cmFwcGVyCgogICAgcmV0dXJuIGRlY29yYXRvcgoKCkBvcGVuX21waV9oYW5kbGVyKHdvcmtlcl9pbnB1dHM9WyJkYXRhX3BhdGgiXSwgcm9vdF93b3JrZXJfaW5wdXRzPXsidmVyYm9zZSI6IFRydWV9KQpkZWYgZGlhcml6ZSgKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBMaXN0W3N0cl1dLAogICAgbW9kZWxfbmFtZTogc3RyID0gInB5YW5ub3RlL3NwZWFrZXItZGlhcml6YXRpb24tMy4wIiwKICAgIGFjY2Vzc190b2tlbjogc3RyID0gTm9uZSwKICAgIGRldmljZTogc3RyID0gTm9uZSwKICAgIHNwZWFrZXJzX2xhYmVsczogTGlzdFtzdHJdID0gTm9uZSwKICAgIHNwZWFrZXJfcHJlZml4OiBzdHIgPSAic3BlYWtlcl8iLAogICAgc2VwYXJhdGVfYnlfY2hhbm5lbHM6IGJvb2wgPSBGYWxzZSwKICAgIG1pbmltdW1fc3BlYWtlcnM6IGludCA9IE5vbmUsCiAgICBtYXhpbXVtX3NwZWFrZXJzOiBpbnQgPSBOb25lLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopIC0+IFR1cGxlW0RpY3Rbc3RyLCBMaXN0W1R1cGxlW2Zsb2F0LCBmbG9hdCwgc3RyXV1dLCBEaWN0W3N0ciwgc3RyXV06CiAgICAiIiIKICAgIFBlcmZvcm0gc3BlZWNoIGRpYXJpemF0aW9uIG9uIGdpdmVuIGF1ZGlvIGZpbGVzIHVzaW5nIHB5YW5ub3RlLWF1ZGlvIChodHRwczovL2dpdGh1Yi5jb20vcHlhbm5vdGUvcHlhbm5vdGUtYXVkaW8pLgogICAgVGhlIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIgZGlhcml6YXRpb24gYXMgdmFsdWUuIEEgZGlhcml6YXRpb24gaXMgYSBsaXN0CiAgICBvZiB0dXBsZXM6IChzdGFydCwgZW5kLCBzcGVha2VyX2xhYmVsKS4KCiAgICBUbyB1c2UgdGhlIGBweWFubm90ZS5hdWRpb2AgbW9kZWxzIHlvdSBtdXN0IHBhc3MgYSBIdWdnaW5nZmFjZSB0b2tlbiBhbmQgZ2V0IGFjY2VzcyB0byB0aGUgcmVxdWlyZWQgbW9kZWxzLiBUaGUKICAgIHRva2VuIGNhbiBiZSBwYXNzZWQgaW4gb25lIG9mIHRoZSBmb2xsb3dpbmcgb3B0aW9uczoKCiAgICAqIFVzZSB0aGUgcGFyYW1ldGVyIGBhY2Nlc3NfdG9rZW5gLgogICAgKiBTZXQgYW4gZW52aXJvbm1lbnQgdmFyaWFibGUgbmFtZWQgIkhVR0dJTkdfRkFDRV9IVUJfVE9LRU4iLgogICAgKiBJZiB1c2luZyBNTFJ1biwgeW91IGNhbiBwYXNzIGl0IGFzIGEgc2VjcmV0IG5hbWVkICJIVUdHSU5HX0ZBQ0VfSFVCX1RPS0VOIi4KCiAgICBUbyBnZXQgYWNjZXNzIHRvIHRoZSBtb2RlbHMgb24gSHVnZ2luZ2ZhY2UsIHZpc2l0IHRoZWlyIHBhZ2UuIEZvciBleGFtcGxlLCB0byB1c2UgdGhlIGRlZmF1bHQgZGlhcml6YXRpb24gbW9kZWwgc2V0CiAgICBpbiB0aGlzIGZ1bmN0aW9uICgicHlhbm5vdGUvc3BlYWtlci1kaWFyaXphdGlvbi0zLjAiKSwgeW91IG5lZWQgYWNjZXNzIGZvciB0aGVzZSB0d28gbW9kZWxzOgoKICAgICogaHR0cHM6Ly9odWdnaW5nZmFjZS5jby9weWFubm90ZS9zZWdtZW50YXRpb24tMy4wCiAgICAqIGh0dHBzOi8vaHVnZ2luZ2ZhY2UuY28vcHlhbm5vdGUvc3BlYWtlci1kaWFyaXphdGlvbi0zLjAKCiAgICBOb3RlOiBUbyBjb250cm9sIHRoZSByZWNvZ25pemVkIHNwZWFrZXJzIGluIHRoZSBkaWFyaXphdGlvbiBvdXRwdXQgeW91IGNhbiBjaG9vc2Ugb25lIG9mIHRoZSBmb2xsb3dpbmcgbWV0aG9kczoKCiAgICAqIEZvciBhIGtub3duIHNwZWFrZXJzIGFtb3VudCwgeW91IG1heSBzZXQgc3BlYWtlciBsYWJlbHMgdmlhIHRoZSBgc3BlYWtlcnNfbGFiZWxzYCBwYXJhbWV0ZXIgdGhhdCB3aWxsIGJlIHVzZWQgaW4KICAgICAgdGhlIG9yZGVyIG9mIHNwZWFraW5nIGluIHRoZSBhdWRpbyAoZmlyc3QgcGVyc29uIHNwZWFraW5nIGJlIHRoZSBmaXJzdCBsYWJlbCBpbiB0aGUgbGlzdCkuIEluIGFkZGl0aW9uLCB5b3UgY2FuIGRvCiAgICAgIGRpYXJpemF0aW9uIHBlciBjaGFubmVsIChzZXR0aW5nIHRoZSBwYXJhbWV0ZXIgYHNlcGFyYXRlX2J5X2NoYW5uZWxzYCB0byBUcnVlKS4gRWFjaCBsYWJlbCB3aWxsIGJlIGFzc2lnbmVkIHRvIGEKICAgICAgc3BlY2lmaWMgY2hhbm5lbCBieSBvcmRlciAoZmlyc3QgbGFiZWwgdG8gY2hhbm5lbCAwLCBzZWNvbmQgbGFiZWwgdG8gY2hhbm5lbCAxIGFuZCBzbyBvbikuIE5vdGljZSwgdGhpcyB3aWxsCiAgICAgIGluY3JlYXNlIHJ1bnRpbWUuCiAgICAqIEZvciB1bmtub3duIHNwZWFrZXJzIGFtb3VudCwgeW91IGNhbiBzZXQgdGhlIGBzcGVha2VyX3ByZWZpeGAgcGFyYW1ldGVyIHRvIGFkZCBhIHByZWZpeCBmb3IgZWFjaCBzcGVha2VyIG51bWJlci4KICAgICAgWW91IGNhbiBhbHNvIGhlbHAgdGhlIGRpYXJpemF0aW9uIGJ5IHNldHRpbmcgdGhlIHNwZWFrZXJzIHJhbmdlIHZpYSB0aGUgYHNwZWFrZXJzX2Ftb3VudF9yYW5nZWAgcGFyYW1ldGVyLgoKICAgIDpwYXJhbSBkYXRhX3BhdGg6ICAgICAgICAgICAgQSBkaXJlY3Rvcnkgb2YgdGhlIGF1ZGlvIGZpbGVzLCBhIHNpbmdsZSBmaWxlIG9yIGEgbGlzdCBvZiBmaWxlcyB0byB0cmFuc2NyaWJlLgogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICBPbmUgb2YgdGhlIG9mZmljaWFsIGRpYXJpemF0aW9uIG1vZGVsIG5hbWVzIChyZWZlcnJlZCBhcyBkaWFyaXphdGlvbiBwaXBlbGluZXMpIG9mCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBweWFubm90ZS5hdWRpb2AgSHVnZ2luZ2ZhY2UgcGFnZS4gRGVmYXVsdDogInB5YW5ub3RlL3NwZWFrZXItZGlhcml6YXRpb24tMy4wIi4KICAgIDpwYXJhbSBhY2Nlc3NfdG9rZW46ICAgICAgICAgQW4gYWNjZXNzIHRva2VuIHRvIHBhc3MgZm9yIHVzaW5nIHRoZSBgcHlhbm5vdGUuYXVkaW9gIG1vZGVscy4gSWYgbm90IHByb3ZpZGVkLCBpdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIGxvb2tpbmcgZm9yIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZSAiSFVHR0lOR19GQUNFX0hVQl9UT0tFTiIuIElmIE1MUnVuIGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGF2YWlsYWJsZSwgaXQgd2lsbCBsb29rIGZvciBhIHNlY3JldCAiSFVHR0lOR19GQUNFX0hVQl9UT0tFTiIuCiAgICA6cGFyYW0gZGV2aWNlOiAgICAgICAgICAgICAgIERldmljZSB0byBsb2FkIHRoZSBtb2RlbC4gQ2FuIGJlIG9uZSBvZiB7ImN1ZGEiLCAiY3B1In0uIERlZmF1bHQgd2lsbCBwcmVmZXIgImN1ZGEiIGlmCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGF2YWlsYWJsZS4KICAgIDpwYXJhbSBzcGVha2Vyc19sYWJlbHM6ICAgICAgTGFiZWxzIHRvIHVzZSBmb3IgdGhlIHJlY29nbml6ZWQgc3BlYWtlcnMuIERlZmF1bHQ6IG51bWVyaWMgbGFiZWxzICgwLCAxLCAuLi4pLgogICAgOnBhcmFtIHNlcGFyYXRlX2J5X2NoYW5uZWxzOiBJZiBlYWNoIHNwZWFrZXIgaXMgc3BlYWtpbmcgaW4gYSBzZXBhcmF0ZSBjaGFubmVsLCB5b3UgY2FuIGRpYXJpemUgZWFjaCBjaGFubmVsIGFuZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb21iaW5lIHRoZSByZXN1bHQgaW50byBhIHNpbmdsZSBkaWFyaXphdGlvbi4gRWFjaCBsYWJlbCBzZXQgaW4gdGhlIGBzcGVha2Vyc19sYWJlbHNgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhcmFtZXRlciB3aWxsIGJlIGFzc2lnbmVkIHRvIGEgc3BlY2lmaWMgY2hhbm5lbCBieSBvcmRlci4KICAgIDpwYXJhbSBzcGVha2VyX3ByZWZpeDogICAgICAgQSBwcmVmaXggdG8gYWRkIGZvciB0aGUgc3BlYWtlcnMgbGFiZWxzLiBUaGlzIHBhcmFtZXRlciBpcyBpZ25vcmVkIGlmCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBzcGVha2Vyc19sYWJlbHNgIGlzIG5vdCBOb25lLiBEZWZhdWx0OiAic3BlYWtlciIuCiAgICA6cGFyYW0gbWluaW11bV9zcGVha2VyczogICAgIFNldCB0aGUgbWluaW11bSBleHBlY3RlZCBhbW91bnQgb2Ygc3BlYWtlcnMgdG8gYmUgaW4gdGhlIGF1ZGlvIGZpbGVzLiBUaGlzIHBhcmFtZXRlciBpcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpZ25vcmVkIGlmIGBzcGVha2Vyc19sYWJlbHNgIGlzIG5vdCBOb25lLgogICAgOnBhcmFtIG1heGltdW1fc3BlYWtlcnM6ICAgICBTZXQgdGhlIG1heGltdW0gZXhwZWN0ZWQgYW1vdW50IG9mIHNwZWFrZXJzIHRvIGJlIGluIHRoZSBhdWRpbyBmaWxlcy4gVGhpcyBwYXJhbWV0ZXIgaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaWdub3JlZCBpZiBgc3BlYWtlcnNfbGFiZWxzYCBpcyBub3QgTm9uZS4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgV2hldGhlciB0byBwcmVzZW50IGxvZ3Mgb2YgYSBwcm9ncmVzcyBiYXIgYW5kIGVycm9ycy4gRGVmYXVsdDogVHJ1ZS4KCiAgICA6cmV0dXJuczogQSB0dXBsZSBvZjoKCiAgICAgICAgICAgICAgKiBTcGVlY2ggZGlhcml6YXRpb24gZGljdGlvbmFyeS4KICAgICAgICAgICAgICAqIEEgZGljdGlvbmFyeSBvZiBlcnJvcmVkIGZpbGVzIHRoYXQgd2VyZSBub3QgdHJhbnNjcmliZWQuCiAgICAiIiIKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBHZXQgdGhlIGlucHV0IGF1ZGlvIGZpbGVzIHRvIGRpYXJpemU6CiAgICBpZiBpc2luc3RhbmNlKGRhdGFfcGF0aCwgc3RyKToKICAgICAgICBkYXRhX3BhdGggPSBwYXRobGliLlBhdGgoZGF0YV9wYXRoKS5hYnNvbHV0ZSgpCiAgICAgICAgYXVkaW9fZmlsZXMgPSBfZ2V0X2F1ZGlvX2ZpbGVzKGRhdGFfcGF0aD1kYXRhX3BhdGgpCiAgICBlbHNlOiAgIyBTaG91bGQgYmUgYSBsaXN0IG9mIGZpbGVzLgogICAgICAgIGF1ZGlvX2ZpbGVzID0gZGF0YV9wYXRoCgogICAgIyBHZXQgdGhlIEh1Z2dpbmdmYWNlIGFjY2VzcyB0b2tlbjoKICAgIGFjY2Vzc190b2tlbiA9IF9nZXRfYWNjZXNzX3Rva2VuKHBhcmFtZXRlcj1hY2Nlc3NfdG9rZW4pCiAgICBpZiBhY2Nlc3NfdG9rZW4gaXMgTm9uZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAiQSBIdWdnaW5nZmFjZSBhY2Nlc3MgdG9rZW4gbXVzdCBiZSBwcm92aWRlZCB0byB1c2UgYHB5YW5ub3RlLmF1ZGlvYCBtb2RlbHMuIEFjY2VzcyB0b2tlbiBjYW4gYmUgcGFzc2VkICIKICAgICAgICAgICAgInZpYSBvbmUgb2YgdGhlIGZvbGxvd2luZyBvcHRpb25zOlxuIgogICAgICAgICAgICAiKiBVc2UgdGhlIHBhcmFtZXRlciBgYWNjZXNzX3Rva2VuYC5cbiIKICAgICAgICAgICAgIiogU2V0IGFuIGVudmlyb25tZW50IHZhcmlhYmxlIG5hbWVkICdIVUdHSU5HX0ZBQ0VfSFVCX1RPS0VOJy5cbiIKICAgICAgICAgICAgIiogSWYgdXNpbmcgTUxSdW4sIHlvdSBjYW4gcGFzcyBpdCBhcyBhIHNlY3JldCBuYW1lZCAnSFVHR0lOR19GQUNFX0hVQl9UT0tFTicuIgogICAgICAgICkKCiAgICAjIExvYWQgdGhlIGRpYXJpemF0aW9uIHBpcGVsaW5lOgogICAgcGlwZWxpbmUgPSBweWFubm90ZS5hdWRpby5QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoCiAgICAgICAgY2hlY2twb2ludF9wYXRoPW1vZGVsX25hbWUsIHVzZV9hdXRoX3Rva2VuPWFjY2Vzc190b2tlbgogICAgKQoKICAgICMgU2V0IHRoZSBkZXZpY2U6CiAgICBkZXZpY2UgPSBkZXZpY2Ugb3IgKCJjdWRhIiBpZiB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpIGVsc2UgImNwdSIpCiAgICBpZiBkZXZpY2UgIT0gImNwdSI6CiAgICAgICAgcGlwZWxpbmUudG8odG9yY2guZGV2aWNlKGRldmljZSkpCgogICAgIyBQcmVwYXJlIHRoZSBzdWNjZXNzZXMgZGF0YWZyYW1lIGFuZCBlcnJvcnMgZGljdGlvbmFyeSB0byBiZSByZXR1cm5lZDoKICAgIGRpYXJpemF0aW9ucyA9IHt9CiAgICBlcnJvcnMgPSB7fQoKICAgICMgUHJlcGFyZSB0aGUgZGlhcml6YXRpb24ga2V5d29yZCBhcmd1bWVudHM6CiAgICBkaWFyaXplX2t3YXJncyA9IHt9CiAgICBpZiBzcGVha2Vyc19sYWJlbHM6CiAgICAgICAgZGlhcml6ZV9rd2FyZ3NbIm51bV9zcGVha2VycyJdID0gbGVuKHNwZWFrZXJzX2xhYmVscykKICAgIGVsc2U6CiAgICAgICAgaWYgbWluaW11bV9zcGVha2VyczoKICAgICAgICAgICAgZGlhcml6ZV9rd2FyZ3NbIm1pbl9zcGVha2VycyJdID0gbWluaW11bV9zcGVha2VycwogICAgICAgIGlmIG1heGltdW1fc3BlYWtlcnM6CiAgICAgICAgICAgIGRpYXJpemVfa3dhcmdzWyJtYXhfc3BlYWtlcnMiXSA9IG1heGltdW1fc3BlYWtlcnMKCiAgICAjIEdvIG92ZXIgdGhlIGF1ZGlvIGZpbGVzIGFuZCBkaWFyaXplOgogICAgZm9yIGF1ZGlvX2ZpbGUgaW4gdHFkbSgKICAgICAgICBhdWRpb19maWxlcywgZGVzYz0iRGlhcml6aW5nIiwgdW5pdD0iZmlsZSIsIGRpc2FibGU9bm90IHZlcmJvc2UKICAgICk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICAjIExvYWQgYXVkaW8gZmlsZToKICAgICAgICAgICAgYXVkaW8sIHNhbXBsZV9yYXRlID0gdG9yY2hhdWRpby5sb2FkKHVyaT1hdWRpb19maWxlLCBjaGFubmVsc19maXJzdD1UcnVlKQogICAgICAgICAgICAjIEdldCB0aGUgZGlhcml6YXRpb24gKGlmIHByb3ZpZGVkKToKICAgICAgICAgICAgZGlhcml6YXRpb25zW2F1ZGlvX2ZpbGUubmFtZV0gPSBfZGlhcml6ZSgKICAgICAgICAgICAgICAgIGF1ZGlvPWF1ZGlvLAogICAgICAgICAgICAgICAgc2FtcGxlX3JhdGU9c2FtcGxlX3JhdGUsCiAgICAgICAgICAgICAgICBwaXBlbGluZT1waXBlbGluZSwKICAgICAgICAgICAgICAgIHNwZWFrZXJzX2xhYmVscz1zcGVha2Vyc19sYWJlbHMsCiAgICAgICAgICAgICAgICBzZXBhcmF0ZV9ieV9jaGFubmVscz1zZXBhcmF0ZV9ieV9jaGFubmVscywKICAgICAgICAgICAgICAgIHNwZWFrZXJfcHJlZml4PXNwZWFrZXJfcHJlZml4LAogICAgICAgICAgICAgICAgZGlhcml6ZV9rd2FyZ3M9ZGlhcml6ZV9rd2FyZ3MsCiAgICAgICAgICAgICkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGV4Y2VwdGlvbjoKICAgICAgICAgICAgIyBOb3RlIHRoZSBleGNlcHRpb24gYXMgZXJyb3IgaW4gdGhlIGRpY3Rpb25hcnk6CiAgICAgICAgICAgIGlmIHZlcmJvc2U6CiAgICAgICAgICAgICAgICBfTE9HR0VSLndhcm5pbmcoZiJFcnJvciBpbiBmaWxlOiAne2F1ZGlvX2ZpbGUubmFtZX0nIikKICAgICAgICAgICAgZXJyb3JzW3N0cihhdWRpb19maWxlLm5hbWUpXSA9IHN0cihleGNlcHRpb24pCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgIyBQcmludCB0aGUgaGVhZCBvZiB0aGUgcHJvZHVjZWQgZGF0YWZyYW1lIGFuZCByZXR1cm46CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkRvbmUgKHtsZW4oZGlhcml6YXRpb25zKX0ve2xlbihhdWRpb19maWxlcyl9KVxuIikKICAgIHJldHVybiBkaWFyaXphdGlvbnMsIGVycm9ycwoKCmRlZiBfZ2V0X2F1ZGlvX2ZpbGVzKAogICAgZGF0YV9wYXRoOiBwYXRobGliLlBhdGgsCikgLT4gTGlzdFtwYXRobGliLlBhdGhdOgogICAgIyBDaGVjayBpZiB0aGUgcGF0aCBpcyBvZiBhIGRpcmVjdG9yeSBvciBhIGZpbGU6CiAgICBpZiBkYXRhX3BhdGguaXNfZGlyKCk6CiAgICAgICAgIyBHZXQgYWxsIGZpbGVzIGluc2lkZSB0aGUgZGlyZWN0b3J5OgogICAgICAgIGF1ZGlvX2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgYXVkaW9fZmlsZXMgPSBbZGF0YV9wYXRoXQogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlVucmVjb2duaXplZCBkYXRhIHBhdGguIFRoZSBwYXJhbWV0ZXIgYGRhdGFfcGF0aGAgbXVzdCBiZSBlaXRoZXIgYSBkaXJlY3RvcnkgcGF0aCBvciBhIGZpbGUgcGF0aC4gIgogICAgICAgICAgICBmIkdpdmVuOiB7c3RyKGRhdGFfcGF0aCl9ICIKICAgICAgICApCgogICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgoKZGVmIF9nZXRfYWNjZXNzX3Rva2VuKHBhcmFtZXRlcjogc3RyKSAtPiBzdHI6CiAgICAjIElmIGdpdmVuIGFzIGEgcGFyYW1ldGVyLCByZXR1cm4gaXQ6CiAgICBpZiBwYXJhbWV0ZXI6CiAgICAgICAgcmV0dXJuIHBhcmFtZXRlcgoKICAgICMgT3RoZXJ3aXNlLCBsb29rIGF0IHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZToKICAgIGVudmlyb25tZW50X3ZhcmlhYmxlID0gb3MuZW52aXJvbi5nZXQoIkhVR0dJTkdfRkFDRV9IVUJfVE9LRU4iKQogICAgaWYgZW52aXJvbm1lbnRfdmFyaWFibGU6CiAgICAgICAgcmV0dXJuIGVudmlyb25tZW50X3ZhcmlhYmxlCgogICAgIyBMYXN0bHksIHRyeSBsb29rIGluIHRoZSBzZXQgc2VjcmV0cyBpbiBNTFJ1bjoKICAgIHNlY3JldCA9IE5vbmUKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KCiAgICAgICAgY29udGV4dCA9IG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KG5hbWU9Im1scnVuIikKICAgICAgICBzZWNyZXQgPSBjb250ZXh0LmdldF9zZWNyZXQoa2V5PSJIVUdHSU5HX0ZBQ0VfSFVCX1RPS0VOIikKICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgICAgIHBhc3MKCiAgICByZXR1cm4gc2VjcmV0CgoKZGVmIF9kaWFyaXplKAogICAgYXVkaW86IHRvcmNoLlRlbnNvciwKICAgIHNhbXBsZV9yYXRlOiBpbnQsCiAgICBwaXBlbGluZTogcHlhbm5vdGUuYXVkaW8uUGlwZWxpbmUsCiAgICBzcGVha2Vyc19sYWJlbHM6IExpc3Rbc3RyXSwKICAgIHNlcGFyYXRlX2J5X2NoYW5uZWxzOiBib29sLAogICAgc3BlYWtlcl9wcmVmaXg6IHN0ciwKICAgIGRpYXJpemVfa3dhcmdzOiBkaWN0LAopIC0+IExpc3RbVHVwbGVbZmxvYXQsIGZsb2F0LCBzdHJdXToKICAgICMgSWYgdGhlcmUgaXMgbm8gbmVlZCBmb3Igc2VwYXJhdGlvbiBieSBjaGFubmVscywgd2UgZGlhcml6ZSBhbmQgcmV0dXJuOgogICAgaWYgbm90IHNlcGFyYXRlX2J5X2NoYW5uZWxzOgogICAgICAgICMgRGlhcml6ZToKICAgICAgICBkaWFyaXphdGlvbjogcHlhbm5vdGUuY29yZS5Bbm5vdGF0aW9uID0gcGlwZWxpbmUoCiAgICAgICAgICAgIGZpbGU9eyJ3YXZlZm9ybSI6IGF1ZGlvLCAic2FtcGxlX3JhdGUiOiBzYW1wbGVfcmF0ZX0sICoqZGlhcml6ZV9rd2FyZ3MKICAgICAgICApCiAgICAgICAgIyBWZXJpZnkgc3BlYWtlcnMgbGFiZWxzIChzaG91bGQgbm90IGZhaWwgaGVyZSBhcyB3ZSBzZXQgYG51bV9zcGVha2Vycz1sZW4oc3BlYWtlcnNfbGFiZWxzKWAgd2hlbiBpbmZlcnJpbmcKICAgICAgICAjIHRocm91Z2ggdGhlIHBpcGVsaW5lKToKICAgICAgICBpZiBzcGVha2Vyc19sYWJlbHM6CiAgICAgICAgICAgIGdpdmVuX3NwZWFrZXJzID0gbGVuKHNwZWFrZXJzX2xhYmVscykKICAgICAgICAgICAgZm91bmRfc3BlYWtlcnMgPSBsZW4oc2V0KGRpYXJpemF0aW9uLmxhYmVscygpKSkKICAgICAgICAgICAgaWYgZ2l2ZW5fc3BlYWtlcnMgPCBmb3VuZF9zcGVha2VyczoKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgZiJOb3QgZW5vdWdoIGBzcGVha2Vyc19sYWJlbHNgIHdlcmUgZ2l2ZW4uIEdvdCB7Z2l2ZW5fc3BlYWtlcnN9IGxhYmVscyBidXQgdGhlIGRpYXJpemF0aW9uICIKICAgICAgICAgICAgICAgICAgICBmInJlY29nbml6ZWQge2ZvdW5kX3NwZWFrZXJzfSBzcGVha2Vycy4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgIyBSZXR1cm4gYXMgYSBkaWFyaXphdGlvbiBsaXN0IC0gYSBzb3J0ZWQgbGlzdCBvZiB0dXBsZXMgb2Ygc3RhcnQgdGltZSwgZW5kIHRpbWUgYW5kIGEgbGFiZWwgKHRoZSBkZWZhdWx0IGxhYmVsCiAgICAgICAgIyByZXR1cm5lZCBpcyAiU1BFQUtFUl9pIiBzbyB3ZSB0YWtlIG9ubHkgdGhlIGluZGV4IG91dCBvZiBpdCk6CiAgICAgICAgcmV0dXJuIFsKICAgICAgICAgICAgKAogICAgICAgICAgICAgICAgc2VnbWVudC5zdGFydCwKICAgICAgICAgICAgICAgIHNlZ21lbnQuZW5kLAogICAgICAgICAgICAgICAgc3BlYWtlcnNfbGFiZWxzW2ludChsYWJlbC5zcGxpdCgiXyIpWzFdKV0KICAgICAgICAgICAgICAgIGlmIHNwZWFrZXJzX2xhYmVscwogICAgICAgICAgICAgICAgZWxzZSBmIntzcGVha2VyX3ByZWZpeH17aW50KGxhYmVsLnNwbGl0KCdfJylbMV0pfSIsCiAgICAgICAgICAgICkKICAgICAgICAgICAgZm9yIHNlZ21lbnQsIHRyYWNrLCBsYWJlbCBpbiBkaWFyaXphdGlvbi5pdGVydHJhY2tzKHlpZWxkX2xhYmVsPVRydWUpCiAgICAgICAgXQoKICAgICMgU2VwYXJhdGUgdG8gY2hhbm5lbHMgYW5kIGRpYXJpemUgKHdlIGV4cGVjdCBvbmx5IG9uZSBzcGVha2VyIHBlciBjaGFubmVsKToKICAgIGNoYW5uZWxfZGlhcml6YXRpb25zID0gWwogICAgICAgIF9kaWFyaXplKAogICAgICAgICAgICBhdWRpbz1hdWRpb1tjaGFubmVsXS51bnNxdWVlemUoCiAgICAgICAgICAgICAgICAwCiAgICAgICAgICAgICksICAjIFRha2UgY2hhbm5lbCBhbmQgYWRkIGEgY2hhbm5lbCBkaW1lbnNpb24gdG8gaXQuCiAgICAgICAgICAgIHNhbXBsZV9yYXRlPXNhbXBsZV9yYXRlLAogICAgICAgICAgICBwaXBlbGluZT1waXBlbGluZSwKICAgICAgICAgICAgc3BlYWtlcnNfbGFiZWxzPVsKICAgICAgICAgICAgICAgIHNwZWFrZXJzX2xhYmVsc1tjaGFubmVsXQogICAgICAgICAgICBdLCAgIyBUYWtlIHRoZSBjaGFubmVsJ3MgbGFiZWwgb25seS4KICAgICAgICAgICAgc2VwYXJhdGVfYnlfY2hhbm5lbHM9RmFsc2UsCiAgICAgICAgICAgIHNwZWFrZXJfcHJlZml4PXNwZWFrZXJfcHJlZml4LAogICAgICAgICAgICBkaWFyaXplX2t3YXJncz17Im51bV9zcGVha2VycyI6IDF9LCAgIyBTZXQgdG8gb25lIHNwZWFrZXIuCiAgICAgICAgKQogICAgICAgIGZvciBjaGFubmVsIGluIHJhbmdlKGF1ZGlvLnNoYXBlWzBdKQogICAgXQoKICAgICMgTWVyZ2UgdGhlIGNoYW5uZWwgZGlhcml6YXRpb25zIGludG8gYSBzaW5nbGUgc29ydGVkIGxpc3Q6CiAgICByZXR1cm4gbGlzdChoZWFwcS5tZXJnZSgqY2hhbm5lbF9kaWFyaXphdGlvbnMpKQo= + base_image: mlrun/mlrun-gpu + commands: [] + code_origin: '' + origin_filename: '' + requirements: + - pyannote.audio + - pyannote.core + - torchaudio + - tqdm + entry_points: + open_mpi_handler: + name: open_mpi_handler + doc: '' + parameters: + - name: worker_inputs + type: List[str] + - name: root_worker_inputs + type: Dict[str, Any] + default: null + outputs: + - default: '' + lineno: 61 + decorator: + name: decorator + doc: '' + parameters: + - name: handler + outputs: + - default: '' + lineno: 73 + wrapper: + name: wrapper + doc: '' + parameters: [] + outputs: + - default: '' + lineno: 78 + diarize: + name: diarize + doc: "Perform speech diarization on given audio files using pyannote-audio (https://github.com/pyannote/pyannote-audio).\n\ + The end result is a dictionary with the file names as keys and their diarization\ + \ as value. A diarization is a list\nof tuples: (start, end, speaker_label).\n\ + \nTo use the `pyannote.audio` models you must pass a Huggingface token and\ + \ get access to the required models. The\ntoken can be passed in one of the\ + \ following options:\n\n* Use the parameter `access_token`.\n* Set an environment\ + \ variable named \"HUGGING_FACE_HUB_TOKEN\".\n* If using MLRun, you can pass\ + \ it as a secret named \"HUGGING_FACE_HUB_TOKEN\".\n\nTo get access to the\ + \ models on Huggingface, visit their page. For example, to use the default\ + \ diarization model set\nin this function (\"pyannote/speaker-diarization-3.0\"\ + ), you need access for these two models:\n\n* https://huggingface.co/pyannote/segmentation-3.0\n\ + * https://huggingface.co/pyannote/speaker-diarization-3.0\n\nNote: To control\ + \ the recognized speakers in the diarization output you can choose one of\ + \ the following methods:\n\n* For a known speakers amount, you may set speaker\ + \ labels via the `speakers_labels` parameter that will be used in\n the order\ + \ of speaking in the audio (first person speaking be the first label in the\ + \ list). In addition, you can do\n diarization per channel (setting the parameter\ + \ `separate_by_channels` to True). Each label will be assigned to a\n specific\ + \ channel by order (first label to channel 0, second label to channel 1 and\ + \ so on). Notice, this will\n increase runtime.\n* For unknown speakers amount,\ + \ you can set the `speaker_prefix` parameter to add a prefix for each speaker\ + \ number.\n You can also help the diarization by setting the speakers range\ + \ via the `speakers_amount_range` parameter." + parameters: + - name: data_path + type: Union[str, List[str]] + doc: A directory of the audio files, a single file or a list of files to transcribe. + - name: model_name + type: str + doc: 'One of the official diarization model names (referred as diarization + pipelines) of `pyannote.audio` Huggingface page. Default: "pyannote/speaker-diarization-3.0".' + default: pyannote/speaker-diarization-3.0 + - name: access_token + type: str + doc: An access token to pass for using the `pyannote.audio` models. If not + provided, it will be looking for the environment variable "HUGGING_FACE_HUB_TOKEN". + If MLRun is available, it will look for a secret "HUGGING_FACE_HUB_TOKEN". + default: null + - name: device + type: str + doc: Device to load the model. Can be one of {"cuda", "cpu"}. Default will + prefer "cuda" if available. + default: null + - name: speakers_labels + type: List[str] + doc: 'Labels to use for the recognized speakers. Default: numeric labels (0, + 1, ...).' + default: null + - name: speaker_prefix + type: str + doc: 'A prefix to add for the speakers labels. This parameter is ignored if + `speakers_labels` is not None. Default: "speaker".' + default: speaker_ + - name: separate_by_channels + type: bool + doc: If each speaker is speaking in a separate channel, you can diarize each + channel and combine the result into a single diarization. Each label set + in the `speakers_labels` parameter will be assigned to a specific channel + by order. + default: false + - name: minimum_speakers + type: int + doc: Set the minimum expected amount of speakers to be in the audio files. + This parameter is ignored if `speakers_labels` is not None. + default: null + - name: maximum_speakers + type: int + doc: Set the maximum expected amount of speakers to be in the audio files. + This parameter is ignored if `speakers_labels` is not None. + default: null + - name: verbose + type: bool + doc: 'Whether to present logs of a progress bar and errors. Default: True.' + default: false + outputs: + - doc: 'A tuple of:' + default: '' + lineno: 139 + description: pyannote's speech diarization of audio files + default_handler: diarize + disable_auto_mount: false + clone_target_dir: '' + env: [] + priority_class_name: '' + preemption_mode: prevent + affinity: null + tolerations: null + security_context: {} +verbose: false diff --git a/speech_diarization/item.yaml b/pyannote_audio/item.yaml similarity index 71% rename from speech_diarization/item.yaml rename to pyannote_audio/item.yaml index f49dbc319..603c1a361 100644 --- a/speech_diarization/item.yaml +++ b/pyannote_audio/item.yaml @@ -3,9 +3,9 @@ categories: - Deep Learning - Huggingface - Audio -description: speech diarization of audio files +description: pyannote's speech diarization of audio files doc: '' -example: speech_diarization.ipynb +example: pyannote_audio.ipynb generationDate: 2023-12-03:14-30 hidden: false icon: '' @@ -14,10 +14,10 @@ labels: maintainers: [] marketplaceType: '' mlrunVersion: 1.5.2 -name: speech_diarization +name: pyannote-audio platformVersion: 3.5.3 spec: - filename: speech_diarization.py + filename: pyannote_audio.py handler: diarize image: mlrun/mlrun-gpu kind: job @@ -27,4 +27,4 @@ spec: - torchaudio - tqdm url: '' -version: 2.0.0 +version: 1.0.0 diff --git a/speech_diarization/speech_diarization.ipynb b/pyannote_audio/pyannote_audio.ipynb similarity index 100% rename from speech_diarization/speech_diarization.ipynb rename to pyannote_audio/pyannote_audio.ipynb diff --git a/speech_diarization/speech_diarization.py b/pyannote_audio/pyannote_audio.py similarity index 100% rename from speech_diarization/speech_diarization.py rename to pyannote_audio/pyannote_audio.py diff --git a/speech_diarization/test_speech_diarization.py b/pyannote_audio/test_pyannote_audio.py similarity index 79% rename from speech_diarization/test_speech_diarization.py rename to pyannote_audio/test_pyannote_audio.py index 71a95575a..93da50834 100644 --- a/speech_diarization/test_speech_diarization.py +++ b/pyannote_audio/test_pyannote_audio.py @@ -1,4 +1,5 @@ import os + import mlrun import pytest @@ -6,8 +7,9 @@ @pytest.mark.skipif("HUGGING_FACE_HUB_TOKEN" not in os.environ, reason="no token") def test_speech_diarization(): project = mlrun.new_project("diarization-test2") - speech_diarization = project.set_function(func="speech_diarization.py", name="speech_diarization", - image="mlrun/mlrun") + speech_diarization = project.set_function( + func="./function.yaml", name="speech_diarization", image="mlrun/mlrun" + ) diarize_run = speech_diarization.run( handler="diarize", diff --git a/question_answering/function.yaml b/question_answering/function.yaml index fad891ac9..a33614153 100644 --- a/question_answering/function.yaml +++ b/question_answering/function.yaml @@ -2,7 +2,7 @@ kind: job metadata: name: question-answering tag: '' - hash: 9f9635a21ce5ea490c939297c7cb60f5b21945ab + hash: 90e67d116b256a98da7d5819724e43df01d8b4eb project: '' labels: author: yonish @@ -13,13 +13,15 @@ spec: args: [] image: '' build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgZW51bQppbXBvcnQgbG9nZ2luZwppbXBvcnQgb3BlcmF0b3IKaW1wb3J0IHBhdGhsaWIKZnJvbSBjb2xsZWN0aW9ucyBpbXBvcnQgQ291bnRlcgpmcm9tIGZ1bmN0b29scyBpbXBvcnQgcmVkdWNlLCB3cmFwcwpmcm9tIHR5cGluZyBpbXBvcnQgQW55LCBEaWN0LCBMaXN0LCBUdXBsZSwgVW5pb24KCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHRyYW5zZm9ybWVycwpmcm9tIHRxZG0gaW1wb3J0IHRxZG0KCiMgR2V0IHRoZSBnbG9iYWwgbG9nZ2VyOgpfTE9HR0VSID0gbG9nZ2luZy5nZXRMb2dnZXIoKQoKCmRlZiBfY2hlY2tfbWxydW5fYW5kX29wZW5fbXBpKCkgLT4gVHVwbGVbIm1scnVuLk1MQ2xpZW50Q3R4IiwgIm1waTRweS5NUEkuSW50cmFjb21tIl06CiAgICBnbG9iYWwgX0xPR0dFUgoKICAgIGlzX21waSA9IEZhbHNlCiAgICB0cnk6CiAgICAgICAgaW1wb3J0IG1scnVuCgogICAgICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJtbHJ1biIpCiAgICAgICAgX0xPR0dFUiA9IGNvbnRleHQubG9nZ2VyCiAgICAgICAgaXNfbXBpID0gY29udGV4dC5sYWJlbHMuZ2V0KCJraW5kIiwgImpvYiIpID09ICJtcGlqb2IiCgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnJvbSBtcGk0cHkgaW1wb3J0IE1QSQoKICAgICAgICAgICAgICAgIHJldHVybiBjb250ZXh0LCBNUEkuQ09NTV9XT1JMRAogICAgICAgICAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtcGk0cHlfbm90X2ZvdW5kOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgIlRvIGRpc3RyaWJ1dGUgdGhlIGZ1bmN0aW9uIHVzaW5nIE1MUnVuJ3MgJ21waWpvYicgeW91IG5lZWQgdG8gaGF2ZSBgbXBpNHB5YCBwYWNrYWdlIGluIHlvdXIgIgogICAgICAgICAgICAgICAgICAgICJpbnRlcnByZXRlci4gUGxlYXNlIHJ1biBgcGlwIGluc3RhbGwgbXBpNHB5YCBhbmQgbWFrZSBzdXJlIHlvdSBoYXZlIG9wZW4tbXBpLiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIG1waTRweV9ub3RfZm91bmQKICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yIGFzIG1vZHVsZV9ub3RfZm91bmQ6CiAgICAgICAgaWYgaXNfbXBpOgogICAgICAgICAgICByYWlzZSBtb2R1bGVfbm90X2ZvdW5kCiAgICByZXR1cm4gTm9uZSwgTm9uZQoKCmRlZiBvcGVuX21waV9oYW5kbGVyKAogICAgd29ya2VyX2lucHV0czogTGlzdFtzdHJdLCByb290X3dvcmtlcl9pbnB1dHM6IERpY3Rbc3RyLCBBbnldID0gTm9uZQopOgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICAjIENoZWNrIGZvciBNTFJ1biBhbmQgT3Blbk1QSSBhdmFpbGFiaWxpdHk6CiAgICBjb250ZXh0LCBjb21tID0gX2NoZWNrX21scnVuX2FuZF9vcGVuX21waSgpCgogICAgZGVmIGRlY29yYXRvcihoYW5kbGVyKToKICAgICAgICBpZiBjb21tIGlzIE5vbmUgb3IgY29tbS5HZXRfc2l6ZSgpID09IDE6CiAgICAgICAgICAgIHJldHVybiBoYW5kbGVyCgogICAgICAgIEB3cmFwcyhoYW5kbGVyKQogICAgICAgIGRlZiB3cmFwcGVyKCoqa3dhcmdzKToKICAgICAgICAgICAgIyBHZXQgdGhlIG9wZW4gbXBpIGVudmlyb25tZW50IHByb3BlcnRpZXM6CiAgICAgICAgICAgIHNpemUgPSBjb21tLkdldF9zaXplKCkKICAgICAgICAgICAgcmFuayA9IGNvbW0uR2V0X3JhbmsoKQoKICAgICAgICAgICAgIyBHaXZlIHRoZSBjb3JyZWN0IGNodW5rIG9mIHRoZSB3b3JrZXJzIGlucHV0czoKICAgICAgICAgICAgZm9yIHdvcmtlcl9pbnB1dCBpbiB3b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBrd2FyZ3Nbd29ya2VyX2lucHV0XQogICAgICAgICAgICAgICAgaWYgaW5wdXRfYXJndW1lbnQgaXMgTm9uZToKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgc3RyKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IF9nZXRfdGV4dF9maWxlcygKICAgICAgICAgICAgICAgICAgICAgICAgZGF0YV9wYXRoPXBhdGhsaWIuUGF0aChpbnB1dF9hcmd1bWVudCkuYWJzb2x1dGUoKQogICAgICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGlmIGxlbihpbnB1dF9hcmd1bWVudCkgPCBzaXplOgogICAgICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgICAgIGYiQ2Fubm90IHNwbGl0IHRoZSBpbnB1dCAne3dvcmtlcl9pbnB1dH0nIG9mIGxlbmd0aCB7bGVuKGlucHV0X2FyZ3VtZW50KX0gdG8ge3NpemV9IHdvcmtlcnMuICIKICAgICAgICAgICAgICAgICAgICAgICAgZiJQbGVhc2UgcmVkdWNlIHRoZSBhbW91bnQgb2Ygd29ya2VycyBmb3IgdGhpcyBpbnB1dC4iCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZXZlbl9jaHVua19zaXplID0gbGVuKGlucHV0X2FyZ3VtZW50KSAvLyBzaXplCiAgICAgICAgICAgICAgICBjaHVua19zdGFydCA9IHJhbmsgKiBldmVuX2NodW5rX3NpemUKICAgICAgICAgICAgICAgIGNodW5rX2VuZCA9ICgKICAgICAgICAgICAgICAgICAgICAocmFuayArIDEpICogZXZlbl9jaHVua19zaXplCiAgICAgICAgICAgICAgICAgICAgaWYgcmFuayArIDEgPCBzaXplCiAgICAgICAgICAgICAgICAgICAgZWxzZSBsZW4oaW5wdXRfYXJndW1lbnQpCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgICAgIGYiUmFuayAje3Jhbmt9OiBQcm9jZXNzaW5nIGlucHV0IGNodW5rIG9mICd7d29ya2VyX2lucHV0fScgIgogICAgICAgICAgICAgICAgICAgIGYiZnJvbSBpbmRleCB7Y2h1bmtfc3RhcnR9IHRvIHtjaHVua19lbmR9LiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIGxpc3QpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gaW5wdXRfYXJndW1lbnRbY2h1bmtfc3RhcnQ6Y2h1bmtfZW5kXQogICAgICAgICAgICAgICAgZWxpZiBpc2luc3RhbmNlKGlucHV0X2FyZ3VtZW50LCBwZC5EYXRhRnJhbWUpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gaW5wdXRfYXJndW1lbnQuaWxvY1tjaHVua19zdGFydDpjaHVua19lbmQ6LCA6XQogICAgICAgICAgICAgICAga3dhcmdzW3dvcmtlcl9pbnB1dF0gPSBpbnB1dF9hcmd1bWVudAoKICAgICAgICAgICAgIyBTZXQgdGhlIHJvb3Qgd29ya2VyIG9ubHkgYXJndW1lbnRzOgogICAgICAgICAgICBpZiByYW5rID09IDAgYW5kIHJvb3Rfd29ya2VyX2lucHV0czoKICAgICAgICAgICAgICAgIGt3YXJncy51cGRhdGUocm9vdF93b3JrZXJfaW5wdXRzKQoKICAgICAgICAgICAgIyBSdW4gdGhlIHdvcmtlcjoKICAgICAgICAgICAgb3V0cHV0ID0gaGFuZGxlcigqKmt3YXJncykKCiAgICAgICAgICAgICMgU2VuZCB0aGUgb3V0cHV0IHRvIHRoZSByb290IHJhbmsgKHJhbmsgIzApOgogICAgICAgICAgICBvdXRwdXQgPSBjb21tLmdhdGhlcihvdXRwdXQsIHJvb3Q9MCkKICAgICAgICAgICAgaWYgcmFuayA9PSAwOgogICAgICAgICAgICAgICAgIyBKb2luIHRoZSBvdXRwdXRzOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiQ29sbGVjdGluZyBkYXRhIGZyb20gd29ya2VycyB0byByb290IHdvcmtlci4iKQogICAgICAgICAgICAgICAgZGF0YWZyYW1lID0gcGQuY29uY2F0KG9ianM9W2RmIGZvciBkZiwgXyBpbiBvdXRwdXRdLCBheGlzPTApCiAgICAgICAgICAgICAgICBlcnJvcnNfZGljdGlvbmFyeSA9IHJlZHVjZShvcGVyYXRvci5pb3IsIFtlcnIgZm9yIF8sIGVyciBpbiBvdXRwdXRdLCB7fSkKICAgICAgICAgICAgICAgIHJldHVybiBkYXRhZnJhbWUsIGVycm9yc19kaWN0aW9uYXJ5CiAgICAgICAgICAgIHJldHVybiBOb25lCgogICAgICAgIHJldHVybiB3cmFwcGVyCgogICAgcmV0dXJuIGRlY29yYXRvcgoKCkBvcGVuX21waV9oYW5kbGVyKHdvcmtlcl9pbnB1dHM9WyJkYXRhX3BhdGgiXSwgcm9vdF93b3JrZXJfaW5wdXRzPXsidmVyYm9zZSI6IFRydWV9KQpkZWYgYW5zd2VyX3F1ZXN0aW9ucygKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBMaXN0W3N0cl1dLAogICAgbW9kZWxfbmFtZTogc3RyLAogICAgcXVlc3Rpb25zOiBVbmlvbltMaXN0W3N0cl0sIExpc3RbTGlzdFtzdHJdXV0sCiAgICBkZXZpY2VfbWFwOiBVbmlvbltzdHIsIGRpY3RdID0gTm9uZSwKICAgIG1vZGVsX2t3YXJnczogZGljdCA9IE5vbmUsCiAgICBhdXRvX2dwdHFfZXhsbGFtYV9tYXhfaW5wdXRfbGVuZ3RoOiBpbnQgPSBOb25lLAogICAgdG9rZW5pemVyX25hbWU6IHN0ciA9IE5vbmUsCiAgICB0b2tlbml6ZXJfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIHRleHRfd3JhcHBlcjogVW5pb25bc3RyLCBMaXN0W3N0cl1dID0gIiIsCiAgICBxdWVzdGlvbnNfd3JhcHBlcjogVW5pb25bc3RyLCBMaXN0W3N0cl1dID0gIiIsCiAgICBnZW5lcmF0aW9uX2NvbmZpZzogVW5pb25bRGljdCwgTGlzdFtEaWN0XV0gPSBOb25lLAogICAgcXVlc3Rpb25zX2NvbmZpZzogVW5pb25bRGljdCwgTGlzdFtEaWN0XV0gPSBOb25lLAogICAgYmF0Y2hfc2l6ZTogaW50ID0gMSwKICAgIHF1ZXN0aW9uc19jb2x1bW5zOiBMaXN0W3N0cl0gPSBOb25lLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopIC0+IFR1cGxlW3BkLkRhdGFGcmFtZSwgZGljdF06CiAgICAiIiIKICAgIEFuc3dlciBxdWVzdGlvbnMgd2l0aCBhIGNvbnRleHQgdG8gdGhlIGdpdmVuIHRleHQgZmlsZXMgY29udGVudHMgYnkgYSBwcmV0cmFpbmVkIExMTSBtb2RlbC4gRWFjaCB0ZXh0IGZpbGUgd2lsbCBoYXZlCiAgICB0aGUgZm9sbG93aW5nIHByb21wdCBidWlsdDoKCiAgICBzdGFydCBvZiBgdGV4dF93cmFwcGVyYAogICAgPHRleHQgZmlsZSBjb250ZW50PgogICAgZW5kIG9mIGB0ZXh0X3dyYXBwZXJgCgogICAgc3RhcnQgb2YgYHF1ZXN0aW9uc193cmFwcGVyYAogICAgMS4gPHF1ZXN0aW9uc1swXT4KICAgIDIuIDxxdWVzdGlvbnNbMV0+CiAgICAuLi4KICAgIG4uIDxxdWVzdGlvbnNbbi0xXT4KICAgIGVuZCBvZiBgcXVlc3Rpb25zX3dyYXBwZXJgCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgICAgICAgICAgICAgICAgIEEgcGF0aCB0byBhIGRpcmVjdG9yeSBvZiB0ZXh0IGZpbGVzIG9yIGEgcGF0aCB0byBhIHRleHQgZmlsZSB0byBhc2sKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBxdWVzdGlvbnMgYWJvdXQuCiAgICA6cGFyYW0gbW9kZWxfbmFtZTogICAgICAgICAgICAgICAgICAgICAgICAgVGhlIHByZS10cmFpbmVkIG1vZGVsIG5hbWUgZnJvbSB0aGUgaHVnZ2luZ2ZhY2UgaHViIHRvIHVzZSBmb3IgYXNraW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcXVlc3Rpb25zLgogICAgOnBhcmFtIHF1ZXN0aW9uczogICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBxdWVzdGlvbnMgdG8gYXNrLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgbGlzdCBvZiBsaXN0cyBvZiBxdWVzdGlvbnMgdG8gYXNrIHBlciB0ZXh0IGZpbGUsIGFuZCBkZXZpZGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYnkgcXVlc3Rpb24gZ3JvdXBzLCB0aGUgZ3JvdXBzIGNhbiBiZSBkdGVybWFpbmVkIGJ5IHNpemUgKGluIG9yZGVyIHRvCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYXZvaWQgbGFyZ2UgaW5wdXRzIHRvIHRoZSBsbG0pIG9yIGJ5IHF1ZXN0aW9uaW5nIG1ldGhvZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChyZWd1bGFyIG9yIHBvbGwgbGlrZSBxdWVzdGlvbmluZykuCiAgICA6cGFyYW0gZGV2aWNlX21hcDogICAgICAgICAgICAgICAgICAgICAgICAgQSBtYXAgdG8gdXNlIGZvciBsb2FkaW5nIHRoZSBtb2RlbCBvbiBtdWx0aXBsZSBkZXZpY2VzLgogICAgOnBhcmFtIG1vZGVsX2t3YXJnczogICAgICAgICAgICAgICAgICAgICAgIEtleXdvcmQgYXJndW1lbnRzIHRvIHBhc3MgZm9yIGxvYWRpbmcgdGhlIG1vZGVsIHVzaW5nIEh1Z2dpbmdGYWNlJ3MKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgdHJhbnNmb3JtZXJzLkF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZGAgZnVuY3Rpb24uCiAgICA6cGFyYW0gYXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aDogRm9yIEF1dG9HUFRRIG1vZGVscyB0byBzZXQgYW5kIGV4dGVuZCB0aGUgbW9kZWwncyBpbnB1dCBidWZmZXIgc2l6ZS4KICAgIDpwYXJhbSB0b2tlbml6ZXJfbmFtZTogICAgICAgICAgICAgICAgICAgICBUaGUgdG9rZW5pemVyIG5hbWUgZnJvbSB0aGUgaHVnZ2luZ2ZhY2UgaHViIHRvIHVzZS4gSWYgbm90IGdpdmVuLCB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbCBuYW1lIHdpbGwgYmUgdXNlZC4KICAgIDpwYXJhbSB0b2tlbml6ZXJfa3dhcmdzOiAgICAgICAgICAgICAgICAgICBLZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIGZvciBsb2FkaW5nIHRoZSB0b2tlbml6ZXIgdXNpbmcgSHVnZ2luZ0ZhY2UncwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGB0cmFuc2Zvcm1lcnMuQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWRgIGZ1bmN0aW9uLgogICAgOnBhcmFtIHRleHRfd3JhcHBlcjogICAgICAgICAgICAgICAgICAgICAgIEEgd3JhcHBlciBmb3IgdGhlIGZpbGUncyB0ZXh0LiBXaWxsIGJlIGFkZGVkIGF0IHRoZSBzdGFydCBvZiB0aGUgcHJvbXB0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE11c3QgaGF2ZSBhIHBsYWNlaG9sZGVyICgne30nKSBmb3IgdGhlIHRleHQgb2YgdGhlIGZpbGUuCiAgICA6cGFyYW0gcXVlc3Rpb25zX3dyYXBwZXI6ICAgICAgICAgICAgICAgICAgQSB3cmFwcGVyIGZvciB0aGUgcXVlc3Rpb25zIHJlY2VpdmVkLiBXaWxsIGJlIGFkZGVkIGFmdGVyIHRoZSB0ZXh0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd3JhcHBlciBpbiB0aGUgcHJvbXB0IHRlbXBsYXRlLiBNdXN0IGhhdmUgYSBwbGFjZWhvbGRlciAoJ3t9JykgZm9yIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHF1ZXN0aW9ucy4KICAgIDpwYXJhbSBnZW5lcmF0aW9uX2NvbmZpZzogICAgICAgICAgICAgICAgICBIdWdnaW5nRmFjZSdzIGBHZW5lcmF0aW9uQ29uZmlnYCBrZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIHRvIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBnZW5lcmF0ZWAgbWV0aG9kLgogICAgOnBhcmFtIHF1ZXN0aW9uc19jb25maWc6ICAgICAgICAgICAgICAgICAgIEEgZGljdGlvbmFyeSBvciBsaXN0IG9mIGRpY3Rpb25hcmllcyBjb250YWluaW5nIHNwZWNpZmljIHdheXMgdG8gYW5zd2VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcXVlc3Rpb25zICh1c2luZyBhIHBvbGwgZm9yIGV4YW1wbGUpLCBlYWNoIGRpY3Rpb25hcnkgaW4gdGhlIGxpc3QgaXMgZm9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY29ycmVzcG9uZGluZyBxdWVzdGlvbiBncm91cCBhbmQgZGV0ZXJtaW5lcyB0aGUgcXVlc3Rpb24gYXNraW5nIG1ldGhvZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvciBzYWlkIGdyb3VwLgogICAgOnBhcmFtIGJhdGNoX3NpemU6ICAgICAgICAgICAgICAgICAgICAgICAgIEJhdGNoIHNpemUgZm9yIGluZmVyZW5jZS4KICAgIDpwYXJhbSBxdWVzdGlvbnNfY29sdW1uczogICAgICAgICAgICAgICAgICBDb2x1bW5zIHRvIHVzZSBmb3IgdGhlIGRhdGFmcmFtZSByZXR1cm5lZC4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHByZXNlbnQgbG9ncyBvZiBhIHByb2dyZXNzIGJhciBhbmQgZXJyb3JzLiBEZWZhdWx0OiBUcnVlLgoKCiAgICA6cmV0dXJuczogQSB0dXBsZSBvZjoKCiAgICAgICAgICAgICAgKiBBIGRhdGFmcmFtZSBkYXRhc2V0IG9mIHRoZSBxdWVzdGlvbnMgYW5zd2Vycy4KICAgICAgICAgICAgICAqIEEgZGljdGlvbmFyeSBvZiBlcnJvcmVkIGZpbGVzIHRoYXQgd2VyZSBub3QgaW5mZXJyZWQgb3Igd2VyZSBub3QgYW5zd2VyZWQgcHJvcGVybHkuCiAgICAiIiIKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBTZXQgY29uZmlncyB0byBlbXB0eSBkaWN0IGlmIG5vdCBnaXZlbjoKICAgIGlmIGdlbmVyYXRpb25fY29uZmlnIGlzIE5vbmU6CiAgICAgICAgZ2VuZXJhdGlvbl9jb25maWcgPSB7fQogICAgaWYgcXVlc3Rpb25zX2NvbmZpZyBpcyBOb25lOgogICAgICAgIHF1ZXN0aW9uc19jb25maWcgPSB7fQoKICAgICMgR2V0IHRoZSBpbnB1dCB0ZXh0IGZpbGVzIHRvIHF1ZXN0aW9uOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgdGV4dCBmaWxlcy4iKQogICAgaWYgaXNpbnN0YW5jZShkYXRhX3BhdGgsIHN0cik6CiAgICAgICAgZGF0YV9wYXRoID0gcGF0aGxpYi5QYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQogICAgICAgIHRleHRfZmlsZXMgPSBfZ2V0X3RleHRfZmlsZXMoZGF0YV9wYXRoPWRhdGFfcGF0aCkKICAgIGVsc2U6CiAgICAgICAgdGV4dF9maWxlcyA9IGRhdGFfcGF0aAogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbih0ZXh0X2ZpbGVzKX0gdGV4dCBmaWxlcy4iKQoKICAgICMgR2V0IHRoZSBwcm9tcHQgdGVtcGxhdGU6CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiQ3JlYXRpbmcgcHJvbXB0IHRlbXBsYXRlLiIpCgogICAgIyBPcmdhbml6ZSBxdWVzdGlvbnMgYXMgYSBsaXN0IG9mIGxpc3QsIGFuZCBjb3VudCBudW1iZXIgb2Ygc3ViLWxpc3RzIGZvciBmdXR1cmUgdXNlCiAgICBudW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzID0gMSBpZiBpc2luc3RhbmNlKHF1ZXN0aW9uc1swXSwgc3RyKSBlbHNlIGxlbihxdWVzdGlvbnMpCiAgICBxdWVzdGlvbnMgPSBfdG9fZ3JvdXBfbGlzdCgKICAgICAgICBhcmd1bWVudF92YWx1ZT1xdWVzdGlvbnMsCiAgICAgICAgYXJndW1lbnRfbmFtZT0icXVlc3Rpb25zIiwKICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICkKCiAgICAjIE9yZ2FuaXplIHByb21wdCBwYXJ0cyBhdCBwcm9wZXIgbGVuZ3RoCiAgICB0ZXh0X3dyYXBwZXIgPSBfdG9fZ3JvdXBfbGlzdCgKICAgICAgICBhcmd1bWVudF92YWx1ZT10ZXh0X3dyYXBwZXIsCiAgICAgICAgYXJndW1lbnRfbmFtZT0idGV4dF93cmFwcGVyIiwKICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICkKICAgIHF1ZXN0aW9uc193cmFwcGVyID0gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgYXJndW1lbnRfdmFsdWU9cXVlc3Rpb25zX3dyYXBwZXIsCiAgICAgICAgYXJndW1lbnRfbmFtZT0icXVlc3Rpb25zX3dyYXBwZXIiLAogICAgICAgIGxlbmd0aD1udW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzLAogICAgKQoKICAgICMgQ3JlYXRlIGEgbGlzdCBvZiBwcm9tcHQgYWNjb3JkaW5nIHRvIGdpdmVuIHBhcnRzIGFuZCBxdWVzdGlvbnMKICAgIHByb21wdF90ZW1wbGF0ZSA9IFtdCiAgICBxdWVzdGlvbnMgPSBxdWVzdGlvbnMgaWYgaXNpbnN0YW5jZShxdWVzdGlvbnNbMF0sIGxpc3QpIGVsc2UgW3F1ZXN0aW9uc10KCiAgICAjIEJ1aWxkIGFsbCBwcm9tcHRzCiAgICBmb3IgaSBpbiByYW5nZShudW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzKToKICAgICAgICBwcm9tcHRfdGVtcGxhdGUuYXBwZW5kKAogICAgICAgICAgICBfZ2V0X3Byb21wdF90ZW1wbGF0ZSgKICAgICAgICAgICAgICAgIHRleHRfd3JhcHBlcj10ZXh0X3dyYXBwZXJbaV0sCiAgICAgICAgICAgICAgICBxdWVzdGlvbnNfd3JhcHBlcj1xdWVzdGlvbnNfd3JhcHBlcltpXSwKICAgICAgICAgICAgICAgIHF1ZXN0aW9ucz1xdWVzdGlvbnNbaV0sCiAgICAgICAgICAgICkKICAgICAgICApCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIlByb21wdCB0ZW1wbGF0ZSBjcmVhdGVkOlxuXG57cHJvbXB0X3RlbXBsYXRlfVxuIikKCiAgICAjIEdldCB0aGUgdG90YWwgYW1vdW50IG9mIHF1ZXN0aW9uczoKICAgIHF1ZXN0aW9uc19hbW91bnQgPSBzdW0oW2xlbihzdWJsaXN0KSBmb3Igc3VibGlzdCBpbiBxdWVzdGlvbnNdKQoKICAgICMgR2V0IHRoZSBxdWVzdGlvbnMgY29sdW1uczoKICAgIHF1ZXN0aW9uc19jb2x1bW5zID0gcXVlc3Rpb25zX2NvbHVtbnMgb3IgWwogICAgICAgIGYicXtpfSIgZm9yIGkgaW4gcmFuZ2UoMSwgcXVlc3Rpb25zX2Ftb3VudCArIDEpCiAgICBdCgogICAgIyBDaGVjayBpZiB3ZSBoYXZlIHRoZSBjb3JyZWN0IGFtb3VudCBvZiBxdWVzdGlvbnMgY29sdW1uczoKICAgIGlmIGxlbihxdWVzdGlvbnNfY29sdW1ucykgIT0gcXVlc3Rpb25zX2Ftb3VudDoKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlRoZSBwcm92aWRlZCBxdWVzdGlvbnMgY29sdW1ucyBsZW5ndGggKHtsZW4ocXVlc3Rpb25zX2NvbHVtbnMpfSkgIgogICAgICAgICAgICBmImRvZXMgbm90IG1hdGNoIHRoZSBxdWVzdGlvbnMgYW1vdW50ICh7cXVlc3Rpb25zX2Ftb3VudH0pIgogICAgICAgICkKCiAgICAjIExvYWQgdGhlIGdlbmVyYXRpb24gY29uZmlnOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkxvYWRpbmcgZ2VuZXJhdGlvbiBjb25maWd1cmF0aW9uLiIpCiAgICBnZW5lcmF0aW9uX2NvbmZpZyA9IFsKICAgICAgICB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZygqKihjZmcgb3Ige30pKQogICAgICAgIGZvciBjZmcgaW4gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgICAgIGFyZ3VtZW50X3ZhbHVlPWdlbmVyYXRpb25fY29uZmlnLAogICAgICAgICAgICBhcmd1bWVudF9uYW1lPSJnZW5lcmF0aW9uX2NvbmZpZyIsCiAgICAgICAgICAgIGxlbmd0aD1udW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzLAogICAgICAgICkKICAgIF0KICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiR2VuZXJhdGlvbiBjb25maWd1cmF0aW9uIGxvYWRlZDoge2dlbmVyYXRpb25fY29uZmlnfSIpCgogICAgIyBMb2FkIHRoZSBtb2RlbCBhbmQgdG9rZW5pemVyIGludG8gYSBwaXBlbGluZSBvYmplY3Q6CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkxvYWRpbmcgbW9kZWwgJ3ttb2RlbF9uYW1lfScuIikKICAgIGdlbmVyYXRpb25fcGlwZWxpbmUgPSBfZ2V0X2dlbmVyYXRpb25fcGlwZWxpbmUoCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGRldmljZV9tYXA9ZGV2aWNlX21hcCwKICAgICAgICB0b2tlbml6ZXJfbmFtZT10b2tlbml6ZXJfbmFtZSBvciBtb2RlbF9uYW1lLAogICAgICAgIG1vZGVsX2t3YXJncz1tb2RlbF9rd2FyZ3Mgb3Ige30sCiAgICAgICAgdG9rZW5pemVyX2t3YXJncz10b2tlbml6ZXJfa3dhcmdzIG9yIHt9LAogICAgICAgIGF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGg9YXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aCwKICAgICAgICBiYXRjaF9zaXplPWJhdGNoX3NpemUsCiAgICApCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiTW9kZWwgbG9hZGVkLiIpCgogICAgIyBQcmVwYXJlIHRoZSBzdWNjZXNzZXMgZGF0YWZyYW1lIGFuZCBlcnJvcnMgZGljdGlvbmFyeSB0byBiZSByZXR1cm5lZDoKICAgIHN1Y2Nlc3NlcyA9IFtdCiAgICBlcnJvcnMgPSB7fQoKICAgICMgU3BsaXQgdGhlIGZpbGVzIGludG8gYmF0Y2hlczoKICAgIGZpbGVfYmF0Y2hlcyA9IFsKICAgICAgICB0ZXh0X2ZpbGVzW2kgOiBpICsgYmF0Y2hfc2l6ZV0KICAgICAgICBpZiBpICsgYmF0Y2hfc2l6ZSA8IGxlbih0ZXh0X2ZpbGVzKQogICAgICAgIGVsc2UgdGV4dF9maWxlc1tpOl0KICAgICAgICBmb3IgaSBpbiByYW5nZSgwLCBsZW4odGV4dF9maWxlcyksIGJhdGNoX3NpemUpCiAgICBdCiAgICBxdWVzdGlvbnNfY29uZmlnID0gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgYXJndW1lbnRfdmFsdWU9cXVlc3Rpb25zX2NvbmZpZywKICAgICAgICBhcmd1bWVudF9uYW1lPSJxdWVzdGlvbnNfY29uZmlnIiwKICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICkKCiAgICAjIENyZWF0ZSBhIGxpc3Qgb2YgcXVlc3Rpb24gaGFuZGxlcnMgYWNjb3JkaW5nIHRvIGdpdmVuIGNvbmZpZ3MKICAgIGhhbmRsZXJzID0gW10KICAgIGZvciBjZmcgaW4gcXVlc3Rpb25zX2NvbmZpZzoKICAgICAgICBxdWVzdGlvbl90eXBlID0gY2ZnLnBvcCgidHlwZSIsICJkZWZhdWx0IikKICAgICAgICBoYW5kbGVycy5hcHBlbmQoUVVFU1RJT05fTUFQUElORy5nZXQocXVlc3Rpb25fdHlwZSkoKipjZmcpKQoKICAgICMgR28gb3ZlciB0aGUgYmF0Y2hlcyBvZiB0ZXh0IGZpbGVzIGFuZCBxdWVzdGlvbiB0aGVtOgogICAgZm9yIGZpbGVfYmF0Y2ggaW4gdHFkbSgKICAgICAgICBmaWxlX2JhdGNoZXMsCiAgICAgICAgZGVzYz0iR2VuZXJhdGluZyBhbnN3ZXJzIiwKICAgICAgICB1bml0PWYiZmlsZSAoYmF0Y2ggb2Yge2JhdGNoX3NpemV9KSIsCiAgICAgICAgZGlzYWJsZT1ub3QgdmVyYm9zZSwKICAgICk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICB0b3RhbF9hbnN3ZXJzID0gW1tdIGZvciBfIGluIHJhbmdlKGJhdGNoX3NpemUpXQoKICAgICAgICAgICAgIyBHbyBvdmVyIGFsbCBxdWVzdGlvbiBncm91cCBwZXIgYmF0Y2ggb2YgZG9jdW1lbnRzCiAgICAgICAgICAgIGZvciBxdWVzdGlvbl9ncm91cCBpbiByYW5nZShudW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzKToKICAgICAgICAgICAgICAgIGN1cnJlbnRfcXVlc3Rpb25zX2Ftb3VudCA9IGxlbihxdWVzdGlvbnNbcXVlc3Rpb25fZ3JvdXBdKQoKICAgICAgICAgICAgICAgICMgUmVhZCBiYXRjaCAocmVhZCB0aGUgdGV4dCBmcm9tIHRoZSB0ZXh0IGZpbGVzKToKICAgICAgICAgICAgICAgIGJhdGNoZWRfaW5wdXQgPSBfcmVhZF9maWxlX2JhdGNoKAogICAgICAgICAgICAgICAgICAgIGZpbGVfYmF0Y2g9ZmlsZV9iYXRjaCwKICAgICAgICAgICAgICAgICAgICBwcm9tcHRfdGVtcGxhdGU9cHJvbXB0X3RlbXBsYXRlW3F1ZXN0aW9uX2dyb3VwXSwKICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAjIEFuc3dlciB0aGUgcXVlc3Rpb25zIHdpdGggZWFjaCBxdWVzdGlvbiBoYW5kbGVyOgogICAgICAgICAgICAgICAgYmF0Y2hlZF9hbnN3ZXJzID0gaGFuZGxlcnNbcXVlc3Rpb25fZ3JvdXBdLmFuc3dlcigKICAgICAgICAgICAgICAgICAgICBxdWVzdGlvbnNfYW1vdW50PWN1cnJlbnRfcXVlc3Rpb25zX2Ftb3VudCwKICAgICAgICAgICAgICAgICAgICBiYXRjaGVkX2lucHV0PWJhdGNoZWRfaW5wdXQsCiAgICAgICAgICAgICAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZT1nZW5lcmF0aW9uX3BpcGVsaW5lLAogICAgICAgICAgICAgICAgICAgIGdlbmVyYXRpb25fY29uZmlnPWdlbmVyYXRpb25fY29uZmlnW3F1ZXN0aW9uX2dyb3VwXSwKICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAjIFB1dCB0aGUgYW5zd2VycyBpbiB0aGUgY29ycmVjdCBwbGFjZSBpbiB0aGUgdG90YWwgYW5zd2VycyBsaXN0IGFjY29yZGluZyB0byB0aGUgcGxhY2UgaW4gdGhlIGJhdGNoOgogICAgICAgICAgICAgICAgZm9yIGkgaW4gcmFuZ2UoYmF0Y2hfc2l6ZSk6CiAgICAgICAgICAgICAgICAgICAgdG90YWxfYW5zd2Vyc1tpXS5leHRlbmQoYmF0Y2hlZF9hbnN3ZXJzW2ldKQoKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSBhbnN3ZXJzIGFuZCBhdHRhY2ggdGhlIGZpbGUgbmFtZToKICAgICAgICAgICAgc3VjY2Vzc2VzLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBbZmlsZS5uYW1lLCAqYW5zd2Vyc10KICAgICAgICAgICAgICAgICAgICBmb3IgZmlsZSwgYW5zd2VycyBpbiB6aXAoZmlsZV9iYXRjaCwgdG90YWxfYW5zd2VycykKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIE5vdGUgdGhlIGV4Y2VwdGlvbiBhcyBlcnJvciBpbiB0aGUgZGljdGlvbmFyeToKICAgICAgICAgICAgYmF0Y2hfZmlsZV9uYW1lcyA9ICIsICIuam9pbihbZmlsZS5uYW1lIGZvciBmaWxlIGluIGZpbGVfYmF0Y2hdKQogICAgICAgICAgICBpZiB2ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi53YXJuaW5nKAogICAgICAgICAgICAgICAgICAgIGYiRXJyb3IgaW4gYmF0Y2ggJ3tiYXRjaF9maWxlX25hbWVzfSc6IHtzdHIoZXhjZXB0aW9uKX0iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgIGVycm9yc1tiYXRjaF9maWxlX25hbWVzXSA9IHN0cihleGNlcHRpb24pCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgIyBDb25zdHJ1Y3QgdGhlIGFuc3dlcnMgZGF0YWZyYW1lOgogICAgY29sdW1ucyA9IFsKICAgICAgICAidGV4dF9maWxlIiwKICAgICAgICAqcXVlc3Rpb25zX2NvbHVtbnMsCiAgICBdCgogICAgIyBDcmVhdGUgYSBkYXRhIGZyYW1lIG9mIGFuc3dlcnMgYnkgZmlsZXMKICAgIHN1Y2Nlc3NlcyA9IHBkLkRhdGFGcmFtZSgKICAgICAgICBzdWNjZXNzZXMsCiAgICAgICAgY29sdW1ucz1jb2x1bW5zLAogICAgKQoKICAgICMgUHJpbnQgdGhlIGhlYWQgb2YgdGhlIHByb2R1Y2VkIGRhdGFmcmFtZSBhbmQgcmV0dXJuOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oCiAgICAgICAgICAgIGYiRG9uZSAoe3N1Y2Nlc3Nlcy5zaGFwZVswXX0ve2xlbih0ZXh0X2ZpbGVzKX0pXG4iCiAgICAgICAgICAgIGYiQW5zd2VycyBzdW1tYXJ5OlxuIgogICAgICAgICAgICBmIntzdWNjZXNzZXMuaGVhZCgpfSIKICAgICAgICApCiAgICByZXR1cm4gc3VjY2Vzc2VzLCBlcnJvcnMKCgpkZWYgX2dldF90ZXh0X2ZpbGVzKAogICAgZGF0YV9wYXRoOiBwYXRobGliLlBhdGgsCikgLT4gTGlzdFtwYXRobGliLlBhdGhdOgoKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgoKICAgICAgICAjIEdldCBhbGwgZmlsZXMgaW5zaWRlIHRoZSBkaXJlY3Rvcnk6CiAgICAgICAgdGV4dF9maWxlcyA9IGxpc3QoZGF0YV9wYXRoLmdsb2IoIiouKiIpKQogICAgZWxpZiBkYXRhX3BhdGguaXNfZmlsZSgpOgogICAgICAgIHRleHRfZmlsZXMgPSBbZGF0YV9wYXRoXQogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlVucmVjb2duaXplZCBkYXRhIHBhdGguIFRoZSBwYXJhbWV0ZXIgYGRhdGFfcGF0aGAgbXVzdCBiZSBlaXRoZXIgYSBkaXJlY3RvcnkgcGF0aCBvciBhIGZpbGUgcGF0aC4gIgogICAgICAgICAgICBmIkdpdmVuOiB7c3RyKGRhdGFfcGF0aCl9ICIKICAgICAgICApCgogICAgcmV0dXJuIHRleHRfZmlsZXMKCgpkZWYgX2dldF9wcm9tcHRfdGVtcGxhdGUoCiAgICB0ZXh0X3dyYXBwZXI6IHN0ciwKICAgIHF1ZXN0aW9uc193cmFwcGVyOiBzdHIsCiAgICBxdWVzdGlvbnM6IExpc3Rbc3RyXSwKKSAtPiBzdHI6CgogICAgIyBWYWxpZGF0ZSBhbmQgYnVpbGQgdGhlIHRleHQgd3JhcHBlcjoKICAgIHRleHRfd3JhcHBlciA9IHRleHRfd3JhcHBlciBvciAoCiAgICAgICAgIkdpdmVuIHRoZSBmb2xsb3dpbmcgdGV4dDpcbiIgIi0tLS0tXG4iICJ7fVxuIiAiLS0tLS0iCiAgICApCiAgICBpZiB0ZXh0X3dyYXBwZXIuY291bnQoInt9IikgIT0gMToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAiVGhlIGB0ZXh0X3dyYXBwZXJgIG11c3QgaW5jbHVkZSBvbmUgcGxhY2Vob2xkZXIgJ3t9JyBmb3IgdGhlIHRleHQgb2YgdGhlIGZpbGUgdG8gYmUgYXNrZWQgYWJvdXQuIgogICAgICAgICkKCiAgICAjIFZhbGlkYXRlIGFuZCBidWlsZCB0aGUgcXVlc3Rpb24gd3JhcHBlcjoKICAgIHF1ZXN0aW9uc193cmFwcGVyID0gcXVlc3Rpb25zX3dyYXBwZXIgb3IgIkFuc3dlciB0aGUgcXVlc3Rpb25zOlxuIiAie30iCiAgICBpZiBxdWVzdGlvbnNfd3JhcHBlci5jb3VudCgie30iKSAhPSAxOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICJUaGUgYHF1ZXN0aW9uc193cmFwcGVyYCBtdXN0IGluY2x1ZGUgb25lIHBsYWNlaG9sZGVyICd7fScgZm9yIHRoZSBsaXN0IG9mIHF1ZXN0aW9ucy4iCiAgICAgICAgKQoKICAgICMgVmFsaWRhdGUgYW5kIHBhcnNlIHRoZSBxdWVzdGlvbnM6CiAgICBpZiBsZW4ocXVlc3Rpb25zKSA9PSAwOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIlBsZWFzZSBpbmNsdWRlIGF0IGxlYXN0IG9uZSBxdWVzdGlvbi4iKQogICAgcXVlc3Rpb25zID0gIlxuIi5qb2luKAogICAgICAgIFtmIntpfS4ge3F1ZXN0aW9ufSIgZm9yIGksIHF1ZXN0aW9uIGluIGVudW1lcmF0ZShxdWVzdGlvbnMsIDEpXQogICAgKQoKICAgICMgQ29uc3RydWN0IHRoZSB0ZW1wbGF0ZToKICAgIHJldHVybiBmInt0ZXh0X3dyYXBwZXJ9XG57cXVlc3Rpb25zX3dyYXBwZXIuZm9ybWF0KHF1ZXN0aW9ucyl9XG4iCgoKZGVmIF9nZXRfZ2VuZXJhdGlvbl9waXBlbGluZSgKICAgIG1vZGVsX25hbWU6IHN0ciwKICAgIGRldmljZV9tYXA6IFVuaW9uW3N0ciwgZGljdF0sCiAgICB0b2tlbml6ZXJfbmFtZTogc3RyLAogICAgbW9kZWxfa3dhcmdzOiBkaWN0LAogICAgdG9rZW5pemVyX2t3YXJnczogZGljdCwKICAgIGF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGg6IGludCA9IE5vbmUsCiAgICBiYXRjaF9zaXplOiBpbnQgPSAxLAopOgogICAgIyBMb2FkIHRoZSBtb2RlbDoKICAgIG1vZGVsID0gdHJhbnNmb3JtZXJzLkF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCgKICAgICAgICBtb2RlbF9uYW1lLCBkZXZpY2VfbWFwPWRldmljZV9tYXAsICoqbW9kZWxfa3dhcmdzCiAgICApCgogICAgIyBTZXQgZXhsbGFtYSBtYXggaW5wdXQgbGVuZ3RoIGlmIHByb3ZpZGVkOgogICAgIyBUaGlzIGNoYW5nZXMgdGhlIG1vZGVsJ3MgY29udGV4dCBzaXplLgogICAgaWYgYXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aDoKICAgICAgICBmcm9tIGF1dG9fZ3B0cSBpbXBvcnQgZXhsbGFtYV9zZXRfbWF4X2lucHV0X2xlbmd0aAoKICAgICAgICBtb2RlbCA9IGV4bGxhbWFfc2V0X21heF9pbnB1dF9sZW5ndGgoCiAgICAgICAgICAgIG1vZGVsPW1vZGVsLCBtYXhfaW5wdXRfbGVuZ3RoPWF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGgKICAgICAgICApCgogICAgIyBMb2FkIHRoZSB0b2tlbml6ZXI6CiAgICB0b2tlbml6ZXIgPSB0cmFuc2Zvcm1lcnMuQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoCiAgICAgICAgdG9rZW5pemVyX25hbWUsICoqdG9rZW5pemVyX2t3YXJncwogICAgKQoKICAgICMgSW5pdGlhbGl6ZSBhIGdlbmVyYXRpb24gcGlwbGluZSBhbmQgcmV0dXJuOgogICAgcGlwZSA9IHRyYW5zZm9ybWVycy5waXBlbGluZSgKICAgICAgICB0YXNrPSJ0ZXh0LWdlbmVyYXRpb24iLAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIHRva2VuaXplcj10b2tlbml6ZXIsCiAgICAgICAgYmF0Y2hfc2l6ZT1iYXRjaF9zaXplLAogICAgKQogICAgcGlwZS50b2tlbml6ZXIucGFkX3Rva2VuX2lkID0gbW9kZWwuY29uZmlnLmVvc190b2tlbl9pZAogICAgcmV0dXJuIHBpcGUKCgpkZWYgX3JlYWRfZmlsZV9iYXRjaCgKICAgIGZpbGVfYmF0Y2g6IExpc3RbcGF0aGxpYi5QYXRoXSwKICAgIHByb21wdF90ZW1wbGF0ZTogc3RyLAopIC0+IExpc3Rbc3RyXToKICAgIGJhdGNoID0gW10KCiAgICAjIEdvIG92ZXIgYWxsIGZpbGVzIGFuZCByZWFkIGluIHVzYWJsZSBmb3JtYXQKICAgIGZvciBmaWxlIGluIGZpbGVfYmF0Y2g6CiAgICAgICAgd2l0aCBvcGVuKGZpbGUsICJyIiwgZW5jb2Rpbmc9InV0Zi04IikgYXMgZnA6CiAgICAgICAgICAgIGJhdGNoLmFwcGVuZChwcm9tcHRfdGVtcGxhdGUuZm9ybWF0KGZwLnJlYWQoKSkpCiAgICByZXR1cm4gYmF0Y2gKCgpkZWYgX3RvX2dyb3VwX2xpc3QoYXJndW1lbnRfdmFsdWU6IGxpc3QsIGFyZ3VtZW50X25hbWU6IHN0ciwgbGVuZ3RoOiBpbnQpOgoKICAgICMgQ2hlY2sgaWYgaXMgbGlzdCwgdHVybiB0byBsaXN0IGlmIG5vdAogICAgYXJndW1lbnRfdmFsdWUgPSAoCiAgICAgICAgYXJndW1lbnRfdmFsdWUgaWYgaXNpbnN0YW5jZShhcmd1bWVudF92YWx1ZSwgbGlzdCkgZWxzZSBbYXJndW1lbnRfdmFsdWVdCiAgICApCiAgICBsaXN0X2xlbiA9IGxlbihhcmd1bWVudF92YWx1ZSkKCiAgICAjIElmIG5vdCBhIGxpc3QsIG9yIGlzIGEgbGlzdCBvZiBsZW4gMSB3ZSBkdXBsaWNhdGUgZm9yIGNvcnJlY3QgbGVuZ3RoCiAgICAjIElmIGxpc3QgaW4gd3JvbmcgbGVuZ3RoIHRocm93IGFuIGVycm9yCiAgICBpZiBsaXN0X2xlbiAhPSBsZW5ndGg6CiAgICAgICAgaWYgbGlzdF9sZW4gPT0gMToKICAgICAgICAgICAgcmV0dXJuIGFyZ3VtZW50X3ZhbHVlICogbGVuZ3RoCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJUaGUgYXJndW1lbnQgdmFsdWUgb2YgJ3thcmd1bWVudF9uYW1lfScgaXMgbm90IGVxdWFsIHRvIHRoZSBsZW5ndGggb2YgdGhlIGdpdmVuIHF1ZXN0aW9ucyAtIHtsZW5ndGh9IgogICAgICAgICkKICAgIHJldHVybiBhcmd1bWVudF92YWx1ZQoKCmNsYXNzIFF1ZXN0aW9uSGFuZGxlcjoKICAgICIiIgogICAgQSBjbGFzcyBmb3IgaGFuZGxpbmcgcXVlc3Rpb25zIGFuc3dlcmluZyBmb3IgYSBnaXZlbiBxdWVzdGlvbiB0eXBlLgogICAgVGhpcyBjbGFzcyBpcyB1c2VkIGFzIGEgYmFzZSBjbGFzcyBmb3IgYWxsIHF1ZXN0aW9uIHR5cGVzLCBhbmQgZm9yIGRlZmF1bHQgcXVlc3Rpb24gdHlwZSAocmVndWxhciBxdWVzdGlvbgogICAgYW5zd2VyaW5nIHdpdGhvdXQgYW55IHNwZWNpYWwgaGFuZGxpbmcpLgogICAgIiIiCgogICAgY2xhc3MgQ29uZmlnS2V5czoKICAgICAgICBwYXNzCgogICAgZGVmIF9faW5pdF9fKHNlbGYsICoqa3dhcmdzKToKICAgICAgICBwYXNzCgogICAgQHN0YXRpY21ldGhvZAogICAgZGVmIF9nZXRfYW5zd2VycyhnZW5lcmF0ZWRfdGV4dDogc3RyLCBxdWVzdGlvbnNfYW1vdW50OiBpbnQpIC0+IExpc3Rbc3RyXToKCiAgICAgICAgIyBDbGVhciBhbnN3ZXIgc3RhcnQgKHBhcnQgYmVmb3JlIG51bWJlcnMpOgogICAgICAgICMgVE9ETyBmaW5kIGJldHRlciB3YXkgdG8gdmVyaWZ5LCBmb3IgbGlzdCBvZiBxdWVzdGlvbnMgdGhpcyBpcyByZWR1bmRhbnQgZm9yIGV4YW1wbGUKICAgICAgICBpZiAiMS4iIG5vdCBpbiBnZW5lcmF0ZWRfdGV4dDoKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgIGYiQW5zd2VyIDEuIGlzIG1pc3NpbmcgZnJvbSB0aGUgZ2VuZXJhdGVkIHRleHQ6ICd7Z2VuZXJhdGVkX3RleHR9JyIKICAgICAgICAgICAgKQogICAgICAgIHRleHQgPSBnZW5lcmF0ZWRfdGV4dC5zcGxpdCgiMS4iLCAxKVsxXQoKICAgICAgICAjIFN0YXJ0IGV4dHJhY3RpbmcgdGhlIGFuc3dlcnM6CiAgICAgICAgYW5zd2VycyA9IFtdCiAgICAgICAgZm9yIGkgaW4gcmFuZ2UoMSwgcXVlc3Rpb25zX2Ftb3VudCArIDEpOgogICAgICAgICAgICAjIElmIGl0J3MgdGhlIGxhc3QgYW5zd2VyIHRvIGxvb2sgZm9yLCB0YWtlIHRoZSByZXN0IG9mIHRoZSB0ZXh0OgogICAgICAgICAgICBpZiBpID09IHF1ZXN0aW9uc19hbW91bnQ6CiAgICAgICAgICAgICAgICBhbnN3ZXJfaSA9IHRleHQKICAgICAgICAgICAgIyBWZXJpZnkgdGhlcmUgaXMgYSBxdWVzdGlvbiBudW1iZXIgaW4gdGhlIHRleHQ6CiAgICAgICAgICAgIGVsaWYgZiJ7aSArIDF9LiIgbm90IGluIHRleHQ6CiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgICAgIGYiQW5zd2VyIHtpICsgMX0uIGlzIG1pc3NpbmcgZnJvbSB0aGUgZ2VuZXJhdGVkIHRleHQ6ICd7Z2VuZXJhdGVkX3RleHR9JyIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgIyBUYWtlIGkncyBhbnN3ZXI6CiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBhbnN3ZXJfaSwgdGV4dCA9IHRleHQuc3BsaXQoZiJ7aSArIDF9LiIsIDEpCiAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgYW5zd2VyIHJlbW92aW5nIHJlZHVuZGFudCBzcGFjZXM6CiAgICAgICAgICAgIGFuc3dlcnMuYXBwZW5kKGFuc3dlcl9pLnN0cmlwKCkpCgogICAgICAgIHJldHVybiBhbnN3ZXJzCgogICAgZGVmIF9pbmZlcl9xdWVzdGlvbnMoCiAgICAgICAgc2VsZiwKICAgICAgICBxdWVzdGlvbnNfYW1vdW50OiBpbnQsCiAgICAgICAgYmF0Y2hlZF9pbnB1dDogTGlzdFtzdHJdLAogICAgICAgIGdlbmVyYXRpb25fcGlwZWxpbmU6IHRyYW5zZm9ybWVycy5QaXBlbGluZSwKICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZzogdHJhbnNmb3JtZXJzLkdlbmVyYXRpb25Db25maWcsCiAgICApIC0+IExpc3RbTGlzdFtzdHJdXToKCiAgICAgICAgIyBJbmZlciB0aHJvdWdoIHRoZSBsbG06CiAgICAgICAgYmF0Y2hlZF9vdXRwdXQgPSBnZW5lcmF0aW9uX3BpcGVsaW5lKAogICAgICAgICAgICBiYXRjaGVkX2lucHV0LAogICAgICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZz1nZW5lcmF0aW9uX2NvbmZpZywKICAgICAgICAgICAgZW9zX3Rva2VuX2lkPWdlbmVyYXRpb25fcGlwZWxpbmUudG9rZW5pemVyLmVvc190b2tlbl9pZCwKICAgICAgICAgICAgcmV0dXJuX2Z1bGxfdGV4dD1GYWxzZSwKICAgICAgICAgICAgbnVtX3JldHVybl9zZXF1ZW5jZXM9MSwKICAgICAgICApCgogICAgICAgICMgUHJvY2VzcyB0aGUgb3V0cHV0cyB0byBnZXQgdGhlIGFuc3dlcnM6CiAgICAgICAgYmF0Y2hlZF9hbnN3ZXJzID0gW10KICAgICAgICBmb3Igb3V0cHV0IGluIGJhdGNoZWRfb3V0cHV0OgogICAgICAgICAgICAjIEdldCB0aGUgZ2VuZXJhdGVkIGFuc3dlcnM6CiAgICAgICAgICAgIGFuc3dlcnMgPSBzZWxmLl9nZXRfYW5zd2VycygKICAgICAgICAgICAgICAgIGdlbmVyYXRlZF90ZXh0PW91dHB1dFswXVsiZ2VuZXJhdGVkX3RleHQiXSwKICAgICAgICAgICAgICAgIHF1ZXN0aW9uc19hbW91bnQ9cXVlc3Rpb25zX2Ftb3VudCwKICAgICAgICAgICAgKQogICAgICAgICAgICAjIENvbGxlY3QgdGhlIHByb2Nlc3NlZCBhbnN3ZXJzOgogICAgICAgICAgICBiYXRjaGVkX2Fuc3dlcnMuYXBwZW5kKGFuc3dlcnMpCiAgICAgICAgcmV0dXJuIGJhdGNoZWRfYW5zd2VycwoKICAgIGRlZiBhbnN3ZXIoCiAgICAgICAgc2VsZiwKICAgICAgICBxdWVzdGlvbnNfYW1vdW50OiBpbnQsCiAgICAgICAgYmF0Y2hlZF9pbnB1dDogTGlzdFtzdHJdLAogICAgICAgIGdlbmVyYXRpb25fcGlwZWxpbmU6IHRyYW5zZm9ybWVycy5QaXBlbGluZSwKICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZzogdHJhbnNmb3JtZXJzLkdlbmVyYXRpb25Db25maWcsCiAgICApIC0+IExpc3RbTGlzdFtzdHJdXToKICAgICAgICAiIiIKICAgICAgICBBbnN3ZXIgcXVlc3Rpb25zIHdpdGggYSBjb250ZXh0IHRvIHRoZSBnaXZlbiB0ZXh0IGZpbGVzIGNvbnRlbnRzIGJ5IGEgcHJldHJhaW5lZCBMTE0gbW9kZWwgaW4gZ2l2ZW4gcGlwZWxpbmUuCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX2luZmVyX3F1ZXN0aW9ucygKICAgICAgICAgICAgcXVlc3Rpb25zX2Ftb3VudD1xdWVzdGlvbnNfYW1vdW50LAogICAgICAgICAgICBiYXRjaGVkX2lucHV0PWJhdGNoZWRfaW5wdXQsCiAgICAgICAgICAgIGdlbmVyYXRpb25fcGlwZWxpbmU9Z2VuZXJhdGlvbl9waXBlbGluZSwKICAgICAgICAgICAgZ2VuZXJhdGlvbl9jb25maWc9Z2VuZXJhdGlvbl9jb25maWcsCiAgICAgICAgKQoKCmNsYXNzIFBvbGxRdWVzdGlvbkhhbmRsZXIoUXVlc3Rpb25IYW5kbGVyKToKICAgICIiIgogICAgU3RhdGljIGNsYXNzIHRvIGhvbGQgYWxsIHRoZSBwb3NzaWJsZSBwb2xsIHF1ZXN0aW9uIGNvbmZpZ3VyYXRpb25zIG9wdGlvbnMga2V5cwogICAgIiIiCgogICAgY2xhc3MgQ29uZmlnS2V5czoKICAgICAgICAiIiIKICAgICAgICBBIGNsYXNzIGZvciBoYW5kbGluZyBxdWVzdGlvbnMgYW5zd2VyaW5nIGZvciBwb2xsIHR5cGUgcXVlc3Rpb25zLgogICAgICAgIFRoZXNlIHR5cGUgb2YgcXVlc3Rpb24gYXJlIGFuc3dlcmVkIGJ5IGFza2luZyB0aGUgc2FtZSBxdWVzdGlvbiBtdWx0aXBsZSB0aW1lcwogICAgICAgIGFuZCBjaG9vc2luZyB0aGUgbW9zdCBjb21tb24gYW5zd2VyIG9yIHRoZSBhdmVyYWdlIGFuc3dlci4KICAgICAgICAiIiIKCiAgICAgICAgIzogVGhlIG51bWJlciBvZiB0aW1lcyB0byBhc2sgdGhlIHNhbWUgcXVlc3Rpb24uCiAgICAgICAgUE9MTF9DT1VOVCA9ICJwb2xsX2NvdW50IgoKICAgICAgICAjOiBUaGUgc3RyYXRlZ3kgdG8gdXNlIGZvciBjaG9vc2luZyB0aGUgYW5zd2VyIGZyb20gdGhlIHBvbGwuCiAgICAgICAgUE9MTF9TVFJBVEVHWSA9ICJwb2xsX3N0cmF0ZWd5IgoKICAgIGNsYXNzIFN0cmF0ZWd5KGVudW0uRW51bSk6CiAgICAgICAgIzogVGhlIG1vc3QgY29tbW9uIGFuc3dlciBzdHJhdGVneS4KICAgICAgICBNT1NUX0NPTU1PTiA9ICJtb3N0X2NvbW1vbiIKCiAgICAgICAgIzogVGhlIGF2ZXJhZ2UgYW5zd2VyIHN0cmF0ZWd5LgogICAgICAgIEFWRVJBR0UgPSAiYXZlcmFnZSIKCiAgICAgICAgQHN0YXRpY21ldGhvZAogICAgICAgIGRlZiBtb3N0X2NvbW1vbihhbnN3ZXJzKToKICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIENhbGN1bGF0ZSB0aGUgbW9zdCBjb21tb24gYW5zd2VyIGZvciBhIGdpdmVuIGxpc3Qgb2YgYW5zd2Vycy4KICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIGNvdW50ID0gQ291bnRlcihhbnN3ZXJzKQogICAgICAgICAgICBtb3N0X2NvbW1vbiA9IGNvdW50Lm1vc3RfY29tbW9uKDEpCiAgICAgICAgICAgIHJldHVybiBtb3N0X2NvbW1vblswXVswXQoKICAgICAgICBAc3RhdGljbWV0aG9kCiAgICAgICAgZGVmIGF2ZXJhZ2UoYW5zd2Vycyk6CiAgICAgICAgICAgICIiIgogICAgICAgICAgICBDYWxjdWxhdGUgdGhlIGF2ZXJhZ2UgYW5zd2VyIGZvciBhIGdpdmVuIGxpc3Qgb2YgYW5zd2Vycy4KICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoYW5zd2Vyc1swXSwgc3RyKToKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgIkNhbm5vdCBwZXJmb3JtIHBvbGwgd2l0aCBhdmVyYWdlIGFuc3dlciBzdHJhdGVneSBvZiBub24gbnVtZXJpYyB2YWx1ZXMsIgogICAgICAgICAgICAgICAgICAgICIgcGxlYXNlIGNoYW5nZSB0aGUgcXVlc3Rpb24gdG8gZ2l2ZSBudW1lcmljIGRhdGEsIG9yIGNob29zZSAnbW9zdF9jb21tb24nIGFzIHN0cmF0ZWd5LiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIG51bWVyaWNfdmFsdWVzID0gYW5zd2VycwogICAgICAgICAgICBhdmcgPSBzdW0obnVtZXJpY192YWx1ZXMpIC8gbGVuKG51bWVyaWNfdmFsdWVzKQoKICAgICAgICAgICAgIyBSb3VuZCB0byB0aGUgY2xvc2VzdCBpbnRlZ2VyIGFuZCByZXR1cm4gY29ycmVzcG9uZGluZyB2YWx1ZQogICAgICAgICAgICByZXR1cm4gcm91bmQoYXZnKQoKICAgICAgICBkZWYgZG8oc2VsZiwgYW5zd2Vycyk6CiAgICAgICAgICAgICIiIgogICAgICAgICAgICBQZXJmb3JtIHRoZSBzdHJhdGVneS4KICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIHJldHVybiBnZXRhdHRyKHNlbGYsIHNlbGYudmFsdWUpKGFuc3dlcnMpCgogICAgZGVmIF9faW5pdF9fKAogICAgICAgIHNlbGYsIHBvbGxfY291bnQ6IGludCA9IDUsIHBvbGxfc3RyYXRlZ3k6IHN0ciA9ICJtb3N0X2NvbW1vbiIpOgogICAgICAgIHN1cGVyKCkuX19pbml0X18oKQogICAgICAgIHNlbGYucG9sbF9jb3VudCA9IHBvbGxfY291bnQKICAgICAgICBzZWxmLnBvbGxfc3RyYXRlZ3kgPSBzZWxmLlN0cmF0ZWd5KHBvbGxfc3RyYXRlZ3kpCgogICAgZGVmIGFuc3dlcigKICAgICAgICBzZWxmLAogICAgICAgIHF1ZXN0aW9uc19hbW91bnQ6IGludCwKICAgICAgICBiYXRjaGVkX2lucHV0OiBMaXN0W3N0cl0sCiAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZTogdHJhbnNmb3JtZXJzLlBpcGVsaW5lLAogICAgICAgIGdlbmVyYXRpb25fY29uZmlnOiB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZywKICAgICkgLT4gTGlzdFtMaXN0W3N0cl1dOgogICAgICAgICIiIgogICAgICAgIEFuc3dlciBxdWVzdGlvbnMgd2l0aCBhIGNvbnRleHQgdG8gdGhlIGdpdmVuIHRleHQgZmlsZXMgY29udGVudHMgYnkgYSBwcmV0cmFpbmVkIExMTSBtb2RlbCBpbiBnaXZlbiBwaXBlbGluZS4KICAgICAgICAiIiIKICAgICAgICByZXR1cm4gc2VsZi5fYW5zd2VyX3BvbGxfcXVlc3Rpb25zKAogICAgICAgICAgICBxdWVzdGlvbnNfYW1vdW50PXF1ZXN0aW9uc19hbW91bnQsCiAgICAgICAgICAgIGJhdGNoZWRfaW5wdXQ9YmF0Y2hlZF9pbnB1dCwKICAgICAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZT1nZW5lcmF0aW9uX3BpcGVsaW5lLAogICAgICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZz1nZW5lcmF0aW9uX2NvbmZpZywKICAgICAgICApCgogICAgZGVmIF9hbnN3ZXJfcG9sbF9xdWVzdGlvbnMoCiAgICAgICAgc2VsZiwKICAgICAgICBxdWVzdGlvbnNfYW1vdW50OiBpbnQsCiAgICAgICAgYmF0Y2hlZF9pbnB1dDogTGlzdFtzdHJdLAogICAgICAgIGdlbmVyYXRpb25fcGlwZWxpbmU6IHRyYW5zZm9ybWVycy5QaXBlbGluZSwKICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZzogdHJhbnNmb3JtZXJzLkdlbmVyYXRpb25Db25maWcsCiAgICApIC0+IExpc3RbTGlzdFtzdHJdXToKICAgICAgICB2b3RlcyA9IFtdCgogICAgICAgICMgUnVuIHRoZSBwb2xsIGZvciBlYWNoIHF1ZXN0aW9uCiAgICAgICAgZm9yIF8gaW4gcmFuZ2Uoc2VsZi5wb2xsX2NvdW50KToKICAgICAgICAgICAgYmF0Y2hlZF9hbnN3ZXJzID0gc2VsZi5faW5mZXJfcXVlc3Rpb25zKAogICAgICAgICAgICAgICAgcXVlc3Rpb25zX2Ftb3VudD1xdWVzdGlvbnNfYW1vdW50LAogICAgICAgICAgICAgICAgYmF0Y2hlZF9pbnB1dD1iYXRjaGVkX2lucHV0LAogICAgICAgICAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZT1nZW5lcmF0aW9uX3BpcGVsaW5lLAogICAgICAgICAgICAgICAgZ2VuZXJhdGlvbl9jb25maWc9Z2VuZXJhdGlvbl9jb25maWcsCiAgICAgICAgICAgICkKICAgICAgICAgICAgdm90ZXMuYXBwZW5kKGJhdGNoZWRfYW5zd2VycykKICAgICAgICBhbnN3ZXJzID0gW10KCiAgICAgICAgIyBDb2xsZWN0IHRoZSBhbnN3ZXJzIGFjY29yZGluZyB0byB0aGUgcG9sbCBzdHJhdGVneQogICAgICAgICMgQXZlcmFnZSBzdHJhdGVneSB3b3JrcyBmb3IgbnVtZXJpYyB2YWx1ZXMgb25seQogICAgICAgIGZvciBiYXRjaCBpbiByYW5nZShsZW4odm90ZXNbMF0pKToKICAgICAgICAgICAgYmF0Y2hlZF9hbnN3ZXJzID0gW10KICAgICAgICAgICAgZm9yIHF1ZXN0aW9uIGluIHJhbmdlKHF1ZXN0aW9uc19hbW91bnQpOgogICAgICAgICAgICAgICAgIyBDcmVhdGUgYSBsaXN0IG9mIGFsbCBhbnN3ZXJzIHRvIHJlbGV2YW50IHF1ZXN0aW9uCiAgICAgICAgICAgICAgICBhbnN3ZXIgPSBbCiAgICAgICAgICAgICAgICAgICAgdm90ZXNbdm90ZXJdW2JhdGNoXVtxdWVzdGlvbl0gZm9yIHZvdGVyIGluIHJhbmdlKHNlbGYucG9sbF9jb3VudCkKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgICAgIGFuc3dlciA9IHNlbGYucG9sbF9zdHJhdGVneS5kbyhhbnN3ZXIpCiAgICAgICAgICAgICAgICBiYXRjaGVkX2Fuc3dlcnMuYXBwZW5kKGFuc3dlcikKICAgICAgICAgICAgYW5zd2Vycy5hcHBlbmQoYmF0Y2hlZF9hbnN3ZXJzKQogICAgICAgIHJldHVybiBhbnN3ZXJzCgoKIyBIb2xkcyBuYW1lcyBvZiBRdWVzdGlvbkhhbmRsZXMKY2xhc3MgUXVlc3Rpb25UeXBlczoKICAgIERFRkFVTFQgPSAiZGVmYXVsdCIKICAgIFBPTEwgPSAicG9sbCIKCgojIE1hcHMgcXVlc3Rpb24gdHlwZXMgdG8gdGhlaXIgaGFuZGxlcnMKUVVFU1RJT05fTUFQUElORyA9IHsKICAgIFF1ZXN0aW9uVHlwZXMuREVGQVVMVDogUXVlc3Rpb25IYW5kbGVyLAogICAgUXVlc3Rpb25UeXBlcy5QT0xMOiBQb2xsUXVlc3Rpb25IYW5kbGVyLAp9Cg== + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgZW51bQppbXBvcnQgbG9nZ2luZwppbXBvcnQgb3BlcmF0b3IKaW1wb3J0IHBhdGhsaWIKZnJvbSBjb2xsZWN0aW9ucyBpbXBvcnQgQ291bnRlcgpmcm9tIGZ1bmN0b29scyBpbXBvcnQgcmVkdWNlLCB3cmFwcwpmcm9tIHR5cGluZyBpbXBvcnQgQW55LCBEaWN0LCBMaXN0LCBUdXBsZSwgVW5pb24KCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHRyYW5zZm9ybWVycwpmcm9tIHRxZG0gaW1wb3J0IHRxZG0KCiMgR2V0IHRoZSBnbG9iYWwgbG9nZ2VyOgpfTE9HR0VSID0gbG9nZ2luZy5nZXRMb2dnZXIoKQoKCmRlZiBfY2hlY2tfbWxydW5fYW5kX29wZW5fbXBpKCkgLT4gVHVwbGVbIm1scnVuLk1MQ2xpZW50Q3R4IiwgIm1waTRweS5NUEkuSW50cmFjb21tIl06CiAgICBnbG9iYWwgX0xPR0dFUgoKICAgIGlzX21waSA9IEZhbHNlCiAgICB0cnk6CiAgICAgICAgaW1wb3J0IG1scnVuCgogICAgICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJtbHJ1biIpCiAgICAgICAgX0xPR0dFUiA9IGNvbnRleHQubG9nZ2VyCiAgICAgICAgaXNfbXBpID0gY29udGV4dC5sYWJlbHMuZ2V0KCJraW5kIiwgImpvYiIpID09ICJtcGlqb2IiCgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnJvbSBtcGk0cHkgaW1wb3J0IE1QSQoKICAgICAgICAgICAgICAgIHJldHVybiBjb250ZXh0LCBNUEkuQ09NTV9XT1JMRAogICAgICAgICAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtcGk0cHlfbm90X2ZvdW5kOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgIlRvIGRpc3RyaWJ1dGUgdGhlIGZ1bmN0aW9uIHVzaW5nIE1MUnVuJ3MgJ21waWpvYicgeW91IG5lZWQgdG8gaGF2ZSBgbXBpNHB5YCBwYWNrYWdlIGluIHlvdXIgIgogICAgICAgICAgICAgICAgICAgICJpbnRlcnByZXRlci4gUGxlYXNlIHJ1biBgcGlwIGluc3RhbGwgbXBpNHB5YCBhbmQgbWFrZSBzdXJlIHlvdSBoYXZlIG9wZW4tbXBpLiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIG1waTRweV9ub3RfZm91bmQKICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yIGFzIG1vZHVsZV9ub3RfZm91bmQ6CiAgICAgICAgaWYgaXNfbXBpOgogICAgICAgICAgICByYWlzZSBtb2R1bGVfbm90X2ZvdW5kCiAgICByZXR1cm4gTm9uZSwgTm9uZQoKCmRlZiBvcGVuX21waV9oYW5kbGVyKAogICAgd29ya2VyX2lucHV0czogTGlzdFtzdHJdLCByb290X3dvcmtlcl9pbnB1dHM6IERpY3Rbc3RyLCBBbnldID0gTm9uZQopOgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICAjIENoZWNrIGZvciBNTFJ1biBhbmQgT3Blbk1QSSBhdmFpbGFiaWxpdHk6CiAgICBjb250ZXh0LCBjb21tID0gX2NoZWNrX21scnVuX2FuZF9vcGVuX21waSgpCgogICAgZGVmIGRlY29yYXRvcihoYW5kbGVyKToKICAgICAgICBpZiBjb21tIGlzIE5vbmUgb3IgY29tbS5HZXRfc2l6ZSgpID09IDE6CiAgICAgICAgICAgIHJldHVybiBoYW5kbGVyCgogICAgICAgIEB3cmFwcyhoYW5kbGVyKQogICAgICAgIGRlZiB3cmFwcGVyKCoqa3dhcmdzKToKICAgICAgICAgICAgIyBHZXQgdGhlIG9wZW4gbXBpIGVudmlyb25tZW50IHByb3BlcnRpZXM6CiAgICAgICAgICAgIHNpemUgPSBjb21tLkdldF9zaXplKCkKICAgICAgICAgICAgcmFuayA9IGNvbW0uR2V0X3JhbmsoKQoKICAgICAgICAgICAgIyBHaXZlIHRoZSBjb3JyZWN0IGNodW5rIG9mIHRoZSB3b3JrZXJzIGlucHV0czoKICAgICAgICAgICAgZm9yIHdvcmtlcl9pbnB1dCBpbiB3b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBrd2FyZ3Nbd29ya2VyX2lucHV0XQogICAgICAgICAgICAgICAgaWYgaW5wdXRfYXJndW1lbnQgaXMgTm9uZToKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgc3RyKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IF9nZXRfdGV4dF9maWxlcygKICAgICAgICAgICAgICAgICAgICAgICAgZGF0YV9wYXRoPXBhdGhsaWIuUGF0aChpbnB1dF9hcmd1bWVudCkuYWJzb2x1dGUoKQogICAgICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGlmIGxlbihpbnB1dF9hcmd1bWVudCkgPCBzaXplOgogICAgICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgICAgIGYiQ2Fubm90IHNwbGl0IHRoZSBpbnB1dCAne3dvcmtlcl9pbnB1dH0nIG9mIGxlbmd0aCB7bGVuKGlucHV0X2FyZ3VtZW50KX0gdG8ge3NpemV9IHdvcmtlcnMuICIKICAgICAgICAgICAgICAgICAgICAgICAgZiJQbGVhc2UgcmVkdWNlIHRoZSBhbW91bnQgb2Ygd29ya2VycyBmb3IgdGhpcyBpbnB1dC4iCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZXZlbl9jaHVua19zaXplID0gbGVuKGlucHV0X2FyZ3VtZW50KSAvLyBzaXplCiAgICAgICAgICAgICAgICBjaHVua19zdGFydCA9IHJhbmsgKiBldmVuX2NodW5rX3NpemUKICAgICAgICAgICAgICAgIGNodW5rX2VuZCA9ICgKICAgICAgICAgICAgICAgICAgICAocmFuayArIDEpICogZXZlbl9jaHVua19zaXplCiAgICAgICAgICAgICAgICAgICAgaWYgcmFuayArIDEgPCBzaXplCiAgICAgICAgICAgICAgICAgICAgZWxzZSBsZW4oaW5wdXRfYXJndW1lbnQpCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgICAgIGYiUmFuayAje3Jhbmt9OiBQcm9jZXNzaW5nIGlucHV0IGNodW5rIG9mICd7d29ya2VyX2lucHV0fScgIgogICAgICAgICAgICAgICAgICAgIGYiZnJvbSBpbmRleCB7Y2h1bmtfc3RhcnR9IHRvIHtjaHVua19lbmR9LiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIGxpc3QpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gaW5wdXRfYXJndW1lbnRbY2h1bmtfc3RhcnQ6Y2h1bmtfZW5kXQogICAgICAgICAgICAgICAgZWxpZiBpc2luc3RhbmNlKGlucHV0X2FyZ3VtZW50LCBwZC5EYXRhRnJhbWUpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gaW5wdXRfYXJndW1lbnQuaWxvY1tjaHVua19zdGFydDpjaHVua19lbmQ6LCA6XQogICAgICAgICAgICAgICAga3dhcmdzW3dvcmtlcl9pbnB1dF0gPSBpbnB1dF9hcmd1bWVudAoKICAgICAgICAgICAgIyBTZXQgdGhlIHJvb3Qgd29ya2VyIG9ubHkgYXJndW1lbnRzOgogICAgICAgICAgICBpZiByYW5rID09IDAgYW5kIHJvb3Rfd29ya2VyX2lucHV0czoKICAgICAgICAgICAgICAgIGt3YXJncy51cGRhdGUocm9vdF93b3JrZXJfaW5wdXRzKQoKICAgICAgICAgICAgIyBSdW4gdGhlIHdvcmtlcjoKICAgICAgICAgICAgb3V0cHV0ID0gaGFuZGxlcigqKmt3YXJncykKCiAgICAgICAgICAgICMgU2VuZCB0aGUgb3V0cHV0IHRvIHRoZSByb290IHJhbmsgKHJhbmsgIzApOgogICAgICAgICAgICBvdXRwdXQgPSBjb21tLmdhdGhlcihvdXRwdXQsIHJvb3Q9MCkKICAgICAgICAgICAgaWYgcmFuayA9PSAwOgogICAgICAgICAgICAgICAgIyBKb2luIHRoZSBvdXRwdXRzOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiQ29sbGVjdGluZyBkYXRhIGZyb20gd29ya2VycyB0byByb290IHdvcmtlci4iKQogICAgICAgICAgICAgICAgZGF0YWZyYW1lID0gcGQuY29uY2F0KG9ianM9W2RmIGZvciBkZiwgXyBpbiBvdXRwdXRdLCBheGlzPTApCiAgICAgICAgICAgICAgICBlcnJvcnNfZGljdGlvbmFyeSA9IHJlZHVjZShvcGVyYXRvci5pb3IsIFtlcnIgZm9yIF8sIGVyciBpbiBvdXRwdXRdLCB7fSkKICAgICAgICAgICAgICAgIHJldHVybiBkYXRhZnJhbWUsIGVycm9yc19kaWN0aW9uYXJ5CiAgICAgICAgICAgIHJldHVybiBOb25lCgogICAgICAgIHJldHVybiB3cmFwcGVyCgogICAgcmV0dXJuIGRlY29yYXRvcgoKCkBvcGVuX21waV9oYW5kbGVyKHdvcmtlcl9pbnB1dHM9WyJkYXRhX3BhdGgiXSwgcm9vdF93b3JrZXJfaW5wdXRzPXsidmVyYm9zZSI6IFRydWV9KQpkZWYgYW5zd2VyX3F1ZXN0aW9ucygKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBMaXN0W3N0cl1dLAogICAgbW9kZWxfbmFtZTogc3RyLAogICAgcXVlc3Rpb25zOiBVbmlvbltMaXN0W3N0cl0sIExpc3RbTGlzdFtzdHJdXV0sCiAgICBkZXZpY2VfbWFwOiBVbmlvbltzdHIsIGRpY3RdID0gTm9uZSwKICAgIG1vZGVsX2t3YXJnczogZGljdCA9IE5vbmUsCiAgICBhdXRvX2dwdHFfZXhsbGFtYV9tYXhfaW5wdXRfbGVuZ3RoOiBpbnQgPSBOb25lLAogICAgdG9rZW5pemVyX25hbWU6IHN0ciA9IE5vbmUsCiAgICB0b2tlbml6ZXJfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIHRleHRfd3JhcHBlcjogVW5pb25bc3RyLCBMaXN0W3N0cl1dID0gIiIsCiAgICBxdWVzdGlvbnNfd3JhcHBlcjogVW5pb25bc3RyLCBMaXN0W3N0cl1dID0gIiIsCiAgICBnZW5lcmF0aW9uX2NvbmZpZzogVW5pb25bRGljdCwgTGlzdFtEaWN0XV0gPSBOb25lLAogICAgcXVlc3Rpb25zX2NvbmZpZzogVW5pb25bRGljdCwgTGlzdFtEaWN0XV0gPSBOb25lLAogICAgYmF0Y2hfc2l6ZTogaW50ID0gMSwKICAgIHF1ZXN0aW9uc19jb2x1bW5zOiBMaXN0W3N0cl0gPSBOb25lLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopIC0+IFR1cGxlW3BkLkRhdGFGcmFtZSwgZGljdF06CiAgICAiIiIKICAgIEFuc3dlciBxdWVzdGlvbnMgd2l0aCBhIGNvbnRleHQgdG8gdGhlIGdpdmVuIHRleHQgZmlsZXMgY29udGVudHMgYnkgYSBwcmV0cmFpbmVkIExMTSBtb2RlbC4gRWFjaCB0ZXh0IGZpbGUgd2lsbCBoYXZlCiAgICB0aGUgZm9sbG93aW5nIHByb21wdCBidWlsdDoKCiAgICBzdGFydCBvZiBgdGV4dF93cmFwcGVyYAogICAgPHRleHQgZmlsZSBjb250ZW50PgogICAgZW5kIG9mIGB0ZXh0X3dyYXBwZXJgCgogICAgc3RhcnQgb2YgYHF1ZXN0aW9uc193cmFwcGVyYAogICAgMS4gPHF1ZXN0aW9uc1swXT4KICAgIDIuIDxxdWVzdGlvbnNbMV0+CiAgICAuLi4KICAgIG4uIDxxdWVzdGlvbnNbbi0xXT4KICAgIGVuZCBvZiBgcXVlc3Rpb25zX3dyYXBwZXJgCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgICAgICAgICAgICAgICAgIEEgcGF0aCB0byBhIGRpcmVjdG9yeSBvZiB0ZXh0IGZpbGVzIG9yIGEgcGF0aCB0byBhIHRleHQgZmlsZSB0byBhc2sKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBxdWVzdGlvbnMgYWJvdXQuCiAgICA6cGFyYW0gbW9kZWxfbmFtZTogICAgICAgICAgICAgICAgICAgICAgICAgVGhlIHByZS10cmFpbmVkIG1vZGVsIG5hbWUgZnJvbSB0aGUgaHVnZ2luZ2ZhY2UgaHViIHRvIHVzZSBmb3IgYXNraW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcXVlc3Rpb25zLgogICAgOnBhcmFtIHF1ZXN0aW9uczogICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBxdWVzdGlvbnMgdG8gYXNrLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgbGlzdCBvZiBsaXN0cyBvZiBxdWVzdGlvbnMgdG8gYXNrIHBlciB0ZXh0IGZpbGUsIGFuZCBkZXZpZGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYnkgcXVlc3Rpb24gZ3JvdXBzLCB0aGUgZ3JvdXBzIGNhbiBiZSBkdGVybWFpbmVkIGJ5IHNpemUgKGluIG9yZGVyIHRvCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYXZvaWQgbGFyZ2UgaW5wdXRzIHRvIHRoZSBsbG0pIG9yIGJ5IHF1ZXN0aW9uaW5nIG1ldGhvZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChyZWd1bGFyIG9yIHBvbGwgbGlrZSBxdWVzdGlvbmluZykuCiAgICA6cGFyYW0gZGV2aWNlX21hcDogICAgICAgICAgICAgICAgICAgICAgICAgQSBtYXAgdG8gdXNlIGZvciBsb2FkaW5nIHRoZSBtb2RlbCBvbiBtdWx0aXBsZSBkZXZpY2VzLgogICAgOnBhcmFtIG1vZGVsX2t3YXJnczogICAgICAgICAgICAgICAgICAgICAgIEtleXdvcmQgYXJndW1lbnRzIHRvIHBhc3MgZm9yIGxvYWRpbmcgdGhlIG1vZGVsIHVzaW5nIEh1Z2dpbmdGYWNlJ3MKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgdHJhbnNmb3JtZXJzLkF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZGAgZnVuY3Rpb24uCiAgICA6cGFyYW0gYXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aDogRm9yIEF1dG9HUFRRIG1vZGVscyB0byBzZXQgYW5kIGV4dGVuZCB0aGUgbW9kZWwncyBpbnB1dCBidWZmZXIgc2l6ZS4KICAgIDpwYXJhbSB0b2tlbml6ZXJfbmFtZTogICAgICAgICAgICAgICAgICAgICBUaGUgdG9rZW5pemVyIG5hbWUgZnJvbSB0aGUgaHVnZ2luZ2ZhY2UgaHViIHRvIHVzZS4gSWYgbm90IGdpdmVuLCB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbCBuYW1lIHdpbGwgYmUgdXNlZC4KICAgIDpwYXJhbSB0b2tlbml6ZXJfa3dhcmdzOiAgICAgICAgICAgICAgICAgICBLZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIGZvciBsb2FkaW5nIHRoZSB0b2tlbml6ZXIgdXNpbmcgSHVnZ2luZ0ZhY2UncwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGB0cmFuc2Zvcm1lcnMuQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWRgIGZ1bmN0aW9uLgogICAgOnBhcmFtIHRleHRfd3JhcHBlcjogICAgICAgICAgICAgICAgICAgICAgIEEgd3JhcHBlciBmb3IgdGhlIGZpbGUncyB0ZXh0LiBXaWxsIGJlIGFkZGVkIGF0IHRoZSBzdGFydCBvZiB0aGUgcHJvbXB0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE11c3QgaGF2ZSBhIHBsYWNlaG9sZGVyICgne30nKSBmb3IgdGhlIHRleHQgb2YgdGhlIGZpbGUuCiAgICA6cGFyYW0gcXVlc3Rpb25zX3dyYXBwZXI6ICAgICAgICAgICAgICAgICAgQSB3cmFwcGVyIGZvciB0aGUgcXVlc3Rpb25zIHJlY2VpdmVkLiBXaWxsIGJlIGFkZGVkIGFmdGVyIHRoZSB0ZXh0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd3JhcHBlciBpbiB0aGUgcHJvbXB0IHRlbXBsYXRlLiBNdXN0IGhhdmUgYSBwbGFjZWhvbGRlciAoJ3t9JykgZm9yIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHF1ZXN0aW9ucy4KICAgIDpwYXJhbSBnZW5lcmF0aW9uX2NvbmZpZzogICAgICAgICAgICAgICAgICBIdWdnaW5nRmFjZSdzIGBHZW5lcmF0aW9uQ29uZmlnYCBrZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIHRvIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBnZW5lcmF0ZWAgbWV0aG9kLgogICAgOnBhcmFtIHF1ZXN0aW9uc19jb25maWc6ICAgICAgICAgICAgICAgICAgIEEgZGljdGlvbmFyeSBvciBsaXN0IG9mIGRpY3Rpb25hcmllcyBjb250YWluaW5nIHNwZWNpZmljIHdheXMgdG8gYW5zd2VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcXVlc3Rpb25zICh1c2luZyBhIHBvbGwgZm9yIGV4YW1wbGUpLCBlYWNoIGRpY3Rpb25hcnkgaW4gdGhlIGxpc3QgaXMgZm9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY29ycmVzcG9uZGluZyBxdWVzdGlvbiBncm91cCBhbmQgZGV0ZXJtaW5lcyB0aGUgcXVlc3Rpb24gYXNraW5nIG1ldGhvZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvciBzYWlkIGdyb3VwLgogICAgOnBhcmFtIGJhdGNoX3NpemU6ICAgICAgICAgICAgICAgICAgICAgICAgIEJhdGNoIHNpemUgZm9yIGluZmVyZW5jZS4KICAgIDpwYXJhbSBxdWVzdGlvbnNfY29sdW1uczogICAgICAgICAgICAgICAgICBDb2x1bW5zIHRvIHVzZSBmb3IgdGhlIGRhdGFmcmFtZSByZXR1cm5lZC4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHByZXNlbnQgbG9ncyBvZiBhIHByb2dyZXNzIGJhciBhbmQgZXJyb3JzLiBEZWZhdWx0OiBUcnVlLgoKCiAgICA6cmV0dXJuczogQSB0dXBsZSBvZjoKCiAgICAgICAgICAgICAgKiBBIGRhdGFmcmFtZSBkYXRhc2V0IG9mIHRoZSBxdWVzdGlvbnMgYW5zd2Vycy4KICAgICAgICAgICAgICAqIEEgZGljdGlvbmFyeSBvZiBlcnJvcmVkIGZpbGVzIHRoYXQgd2VyZSBub3QgaW5mZXJyZWQgb3Igd2VyZSBub3QgYW5zd2VyZWQgcHJvcGVybHkuCiAgICAiIiIKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBTZXQgY29uZmlncyB0byBlbXB0eSBkaWN0IGlmIG5vdCBnaXZlbjoKICAgIGlmIGdlbmVyYXRpb25fY29uZmlnIGlzIE5vbmU6CiAgICAgICAgZ2VuZXJhdGlvbl9jb25maWcgPSB7fQogICAgaWYgcXVlc3Rpb25zX2NvbmZpZyBpcyBOb25lOgogICAgICAgIHF1ZXN0aW9uc19jb25maWcgPSB7fQoKICAgICMgR2V0IHRoZSBpbnB1dCB0ZXh0IGZpbGVzIHRvIHF1ZXN0aW9uOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgdGV4dCBmaWxlcy4iKQogICAgaWYgaXNpbnN0YW5jZShkYXRhX3BhdGgsIHN0cik6CiAgICAgICAgZGF0YV9wYXRoID0gcGF0aGxpYi5QYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQogICAgICAgIHRleHRfZmlsZXMgPSBfZ2V0X3RleHRfZmlsZXMoZGF0YV9wYXRoPWRhdGFfcGF0aCkKICAgIGVsc2U6CiAgICAgICAgdGV4dF9maWxlcyA9IGRhdGFfcGF0aAogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbih0ZXh0X2ZpbGVzKX0gdGV4dCBmaWxlcy4iKQoKICAgICMgR2V0IHRoZSBwcm9tcHQgdGVtcGxhdGU6CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiQ3JlYXRpbmcgcHJvbXB0IHRlbXBsYXRlLiIpCgogICAgIyBPcmdhbml6ZSBxdWVzdGlvbnMgYXMgYSBsaXN0IG9mIGxpc3QsIGFuZCBjb3VudCBudW1iZXIgb2Ygc3ViLWxpc3RzIGZvciBmdXR1cmUgdXNlCiAgICBudW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzID0gMSBpZiBpc2luc3RhbmNlKHF1ZXN0aW9uc1swXSwgc3RyKSBlbHNlIGxlbihxdWVzdGlvbnMpCiAgICBxdWVzdGlvbnMgPSBfdG9fZ3JvdXBfbGlzdCgKICAgICAgICBhcmd1bWVudF92YWx1ZT1xdWVzdGlvbnMsCiAgICAgICAgYXJndW1lbnRfbmFtZT0icXVlc3Rpb25zIiwKICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICkKCiAgICAjIE9yZ2FuaXplIHByb21wdCBwYXJ0cyBhdCBwcm9wZXIgbGVuZ3RoCiAgICB0ZXh0X3dyYXBwZXIgPSBfdG9fZ3JvdXBfbGlzdCgKICAgICAgICBhcmd1bWVudF92YWx1ZT10ZXh0X3dyYXBwZXIsCiAgICAgICAgYXJndW1lbnRfbmFtZT0idGV4dF93cmFwcGVyIiwKICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICkKICAgIHF1ZXN0aW9uc193cmFwcGVyID0gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgYXJndW1lbnRfdmFsdWU9cXVlc3Rpb25zX3dyYXBwZXIsCiAgICAgICAgYXJndW1lbnRfbmFtZT0icXVlc3Rpb25zX3dyYXBwZXIiLAogICAgICAgIGxlbmd0aD1udW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzLAogICAgKQoKICAgICMgQ3JlYXRlIGEgbGlzdCBvZiBwcm9tcHQgYWNjb3JkaW5nIHRvIGdpdmVuIHBhcnRzIGFuZCBxdWVzdGlvbnMKICAgIHByb21wdF90ZW1wbGF0ZSA9IFtdCiAgICBxdWVzdGlvbnMgPSBxdWVzdGlvbnMgaWYgaXNpbnN0YW5jZShxdWVzdGlvbnNbMF0sIGxpc3QpIGVsc2UgW3F1ZXN0aW9uc10KCiAgICAjIEJ1aWxkIGFsbCBwcm9tcHRzCiAgICBmb3IgaSBpbiByYW5nZShudW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzKToKICAgICAgICBwcm9tcHRfdGVtcGxhdGUuYXBwZW5kKAogICAgICAgICAgICBfZ2V0X3Byb21wdF90ZW1wbGF0ZSgKICAgICAgICAgICAgICAgIHRleHRfd3JhcHBlcj10ZXh0X3dyYXBwZXJbaV0sCiAgICAgICAgICAgICAgICBxdWVzdGlvbnNfd3JhcHBlcj1xdWVzdGlvbnNfd3JhcHBlcltpXSwKICAgICAgICAgICAgICAgIHF1ZXN0aW9ucz1xdWVzdGlvbnNbaV0sCiAgICAgICAgICAgICkKICAgICAgICApCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIlByb21wdCB0ZW1wbGF0ZSBjcmVhdGVkOlxuXG57cHJvbXB0X3RlbXBsYXRlfVxuIikKCiAgICAjIEdldCB0aGUgdG90YWwgYW1vdW50IG9mIHF1ZXN0aW9uczoKICAgIHF1ZXN0aW9uc19hbW91bnQgPSBzdW0oW2xlbihzdWJsaXN0KSBmb3Igc3VibGlzdCBpbiBxdWVzdGlvbnNdKQoKICAgICMgR2V0IHRoZSBxdWVzdGlvbnMgY29sdW1uczoKICAgIHF1ZXN0aW9uc19jb2x1bW5zID0gcXVlc3Rpb25zX2NvbHVtbnMgb3IgWwogICAgICAgIGYicXtpfSIgZm9yIGkgaW4gcmFuZ2UoMSwgcXVlc3Rpb25zX2Ftb3VudCArIDEpCiAgICBdCgogICAgIyBDaGVjayBpZiB3ZSBoYXZlIHRoZSBjb3JyZWN0IGFtb3VudCBvZiBxdWVzdGlvbnMgY29sdW1uczoKICAgIGlmIGxlbihxdWVzdGlvbnNfY29sdW1ucykgIT0gcXVlc3Rpb25zX2Ftb3VudDoKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlRoZSBwcm92aWRlZCBxdWVzdGlvbnMgY29sdW1ucyBsZW5ndGggKHtsZW4ocXVlc3Rpb25zX2NvbHVtbnMpfSkgIgogICAgICAgICAgICBmImRvZXMgbm90IG1hdGNoIHRoZSBxdWVzdGlvbnMgYW1vdW50ICh7cXVlc3Rpb25zX2Ftb3VudH0pIgogICAgICAgICkKCiAgICAjIExvYWQgdGhlIGdlbmVyYXRpb24gY29uZmlnOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkxvYWRpbmcgZ2VuZXJhdGlvbiBjb25maWd1cmF0aW9uLiIpCiAgICBnZW5lcmF0aW9uX2NvbmZpZyA9IFsKICAgICAgICB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZygqKihjZmcgb3Ige30pKQogICAgICAgIGZvciBjZmcgaW4gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgICAgIGFyZ3VtZW50X3ZhbHVlPWdlbmVyYXRpb25fY29uZmlnLAogICAgICAgICAgICBhcmd1bWVudF9uYW1lPSJnZW5lcmF0aW9uX2NvbmZpZyIsCiAgICAgICAgICAgIGxlbmd0aD1udW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzLAogICAgICAgICkKICAgIF0KICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiR2VuZXJhdGlvbiBjb25maWd1cmF0aW9uIGxvYWRlZDoge2dlbmVyYXRpb25fY29uZmlnfSIpCgogICAgIyBMb2FkIHRoZSBtb2RlbCBhbmQgdG9rZW5pemVyIGludG8gYSBwaXBlbGluZSBvYmplY3Q6CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkxvYWRpbmcgbW9kZWwgJ3ttb2RlbF9uYW1lfScuIikKICAgIGdlbmVyYXRpb25fcGlwZWxpbmUgPSBfZ2V0X2dlbmVyYXRpb25fcGlwZWxpbmUoCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGRldmljZV9tYXA9ZGV2aWNlX21hcCwKICAgICAgICB0b2tlbml6ZXJfbmFtZT10b2tlbml6ZXJfbmFtZSBvciBtb2RlbF9uYW1lLAogICAgICAgIG1vZGVsX2t3YXJncz1tb2RlbF9rd2FyZ3Mgb3Ige30sCiAgICAgICAgdG9rZW5pemVyX2t3YXJncz10b2tlbml6ZXJfa3dhcmdzIG9yIHt9LAogICAgICAgIGF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGg9YXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aCwKICAgICAgICBiYXRjaF9zaXplPWJhdGNoX3NpemUsCiAgICApCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiTW9kZWwgbG9hZGVkLiIpCgogICAgIyBQcmVwYXJlIHRoZSBzdWNjZXNzZXMgZGF0YWZyYW1lIGFuZCBlcnJvcnMgZGljdGlvbmFyeSB0byBiZSByZXR1cm5lZDoKICAgIHN1Y2Nlc3NlcyA9IFtdCiAgICBlcnJvcnMgPSB7fQoKICAgICMgU3BsaXQgdGhlIGZpbGVzIGludG8gYmF0Y2hlczoKICAgIGZpbGVfYmF0Y2hlcyA9IFsKICAgICAgICB0ZXh0X2ZpbGVzW2kgOiBpICsgYmF0Y2hfc2l6ZV0KICAgICAgICBpZiBpICsgYmF0Y2hfc2l6ZSA8IGxlbih0ZXh0X2ZpbGVzKQogICAgICAgIGVsc2UgdGV4dF9maWxlc1tpOl0KICAgICAgICBmb3IgaSBpbiByYW5nZSgwLCBsZW4odGV4dF9maWxlcyksIGJhdGNoX3NpemUpCiAgICBdCiAgICBxdWVzdGlvbnNfY29uZmlnID0gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgYXJndW1lbnRfdmFsdWU9cXVlc3Rpb25zX2NvbmZpZywKICAgICAgICBhcmd1bWVudF9uYW1lPSJxdWVzdGlvbnNfY29uZmlnIiwKICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICkKCiAgICAjIENyZWF0ZSBhIGxpc3Qgb2YgcXVlc3Rpb24gaGFuZGxlcnMgYWNjb3JkaW5nIHRvIGdpdmVuIGNvbmZpZ3MKICAgIGhhbmRsZXJzID0gW10KICAgIGZvciBjZmcgaW4gcXVlc3Rpb25zX2NvbmZpZzoKICAgICAgICBxdWVzdGlvbl90eXBlID0gY2ZnLnBvcCgidHlwZSIsICJkZWZhdWx0IikKICAgICAgICBoYW5kbGVycy5hcHBlbmQoUVVFU1RJT05fTUFQUElORy5nZXQocXVlc3Rpb25fdHlwZSkoKipjZmcpKQoKICAgICMgR28gb3ZlciB0aGUgYmF0Y2hlcyBvZiB0ZXh0IGZpbGVzIGFuZCBxdWVzdGlvbiB0aGVtOgogICAgZm9yIGZpbGVfYmF0Y2ggaW4gdHFkbSgKICAgICAgICBmaWxlX2JhdGNoZXMsCiAgICAgICAgZGVzYz0iR2VuZXJhdGluZyBhbnN3ZXJzIiwKICAgICAgICB1bml0PWYiZmlsZSAoYmF0Y2ggb2Yge2JhdGNoX3NpemV9KSIsCiAgICAgICAgZGlzYWJsZT1ub3QgdmVyYm9zZSwKICAgICk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICB0b3RhbF9hbnN3ZXJzID0gW1tdIGZvciBfIGluIHJhbmdlKGJhdGNoX3NpemUpXQoKICAgICAgICAgICAgIyBHbyBvdmVyIGFsbCBxdWVzdGlvbiBncm91cCBwZXIgYmF0Y2ggb2YgZG9jdW1lbnRzCiAgICAgICAgICAgIGZvciBxdWVzdGlvbl9ncm91cCBpbiByYW5nZShudW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzKToKICAgICAgICAgICAgICAgIGN1cnJlbnRfcXVlc3Rpb25zX2Ftb3VudCA9IGxlbihxdWVzdGlvbnNbcXVlc3Rpb25fZ3JvdXBdKQoKICAgICAgICAgICAgICAgICMgUmVhZCBiYXRjaCAocmVhZCB0aGUgdGV4dCBmcm9tIHRoZSB0ZXh0IGZpbGVzKToKICAgICAgICAgICAgICAgIGJhdGNoZWRfaW5wdXQgPSBfcmVhZF9maWxlX2JhdGNoKAogICAgICAgICAgICAgICAgICAgIGZpbGVfYmF0Y2g9ZmlsZV9iYXRjaCwKICAgICAgICAgICAgICAgICAgICBwcm9tcHRfdGVtcGxhdGU9cHJvbXB0X3RlbXBsYXRlW3F1ZXN0aW9uX2dyb3VwXSwKICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAjIEFuc3dlciB0aGUgcXVlc3Rpb25zIHdpdGggZWFjaCBxdWVzdGlvbiBoYW5kbGVyOgogICAgICAgICAgICAgICAgYmF0Y2hlZF9hbnN3ZXJzID0gaGFuZGxlcnNbcXVlc3Rpb25fZ3JvdXBdLmFuc3dlcigKICAgICAgICAgICAgICAgICAgICBxdWVzdGlvbnNfYW1vdW50PWN1cnJlbnRfcXVlc3Rpb25zX2Ftb3VudCwKICAgICAgICAgICAgICAgICAgICBiYXRjaGVkX2lucHV0PWJhdGNoZWRfaW5wdXQsCiAgICAgICAgICAgICAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZT1nZW5lcmF0aW9uX3BpcGVsaW5lLAogICAgICAgICAgICAgICAgICAgIGdlbmVyYXRpb25fY29uZmlnPWdlbmVyYXRpb25fY29uZmlnW3F1ZXN0aW9uX2dyb3VwXSwKICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAjIFB1dCB0aGUgYW5zd2VycyBpbiB0aGUgY29ycmVjdCBwbGFjZSBpbiB0aGUgdG90YWwgYW5zd2VycyBsaXN0IGFjY29yZGluZyB0byB0aGUgcGxhY2UgaW4gdGhlIGJhdGNoOgogICAgICAgICAgICAgICAgZm9yIGkgaW4gcmFuZ2UoYmF0Y2hfc2l6ZSk6CiAgICAgICAgICAgICAgICAgICAgdG90YWxfYW5zd2Vyc1tpXS5leHRlbmQoYmF0Y2hlZF9hbnN3ZXJzW2ldKQoKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSBhbnN3ZXJzIGFuZCBhdHRhY2ggdGhlIGZpbGUgbmFtZToKICAgICAgICAgICAgc3VjY2Vzc2VzLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBbZmlsZS5uYW1lLCAqYW5zd2Vyc10KICAgICAgICAgICAgICAgICAgICBmb3IgZmlsZSwgYW5zd2VycyBpbiB6aXAoZmlsZV9iYXRjaCwgdG90YWxfYW5zd2VycykKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIE5vdGUgdGhlIGV4Y2VwdGlvbiBhcyBlcnJvciBpbiB0aGUgZGljdGlvbmFyeToKICAgICAgICAgICAgYmF0Y2hfZmlsZV9uYW1lcyA9ICIsICIuam9pbihbZmlsZS5uYW1lIGZvciBmaWxlIGluIGZpbGVfYmF0Y2hdKQogICAgICAgICAgICBpZiB2ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi53YXJuaW5nKAogICAgICAgICAgICAgICAgICAgIGYiRXJyb3IgaW4gYmF0Y2ggJ3tiYXRjaF9maWxlX25hbWVzfSc6IHtzdHIoZXhjZXB0aW9uKX0iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgIGVycm9yc1tiYXRjaF9maWxlX25hbWVzXSA9IHN0cihleGNlcHRpb24pCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgIyBDb25zdHJ1Y3QgdGhlIGFuc3dlcnMgZGF0YWZyYW1lOgogICAgY29sdW1ucyA9IFsKICAgICAgICAidGV4dF9maWxlIiwKICAgICAgICAqcXVlc3Rpb25zX2NvbHVtbnMsCiAgICBdCgogICAgIyBDcmVhdGUgYSBkYXRhIGZyYW1lIG9mIGFuc3dlcnMgYnkgZmlsZXMKICAgIHN1Y2Nlc3NlcyA9IHBkLkRhdGFGcmFtZSgKICAgICAgICBzdWNjZXNzZXMsCiAgICAgICAgY29sdW1ucz1jb2x1bW5zLAogICAgKQoKICAgICMgUHJpbnQgdGhlIGhlYWQgb2YgdGhlIHByb2R1Y2VkIGRhdGFmcmFtZSBhbmQgcmV0dXJuOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oCiAgICAgICAgICAgIGYiRG9uZSAoe3N1Y2Nlc3Nlcy5zaGFwZVswXX0ve2xlbih0ZXh0X2ZpbGVzKX0pXG4iCiAgICAgICAgICAgIGYiQW5zd2VycyBzdW1tYXJ5OlxuIgogICAgICAgICAgICBmIntzdWNjZXNzZXMuaGVhZCgpfSIKICAgICAgICApCiAgICByZXR1cm4gc3VjY2Vzc2VzLCBlcnJvcnMKCgpkZWYgX2dldF90ZXh0X2ZpbGVzKAogICAgZGF0YV9wYXRoOiBwYXRobGliLlBhdGgsCikgLT4gTGlzdFtwYXRobGliLlBhdGhdOgoKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgoKICAgICAgICAjIEdldCBhbGwgZmlsZXMgaW5zaWRlIHRoZSBkaXJlY3Rvcnk6CiAgICAgICAgdGV4dF9maWxlcyA9IGxpc3QoZGF0YV9wYXRoLmdsb2IoIiouKiIpKQogICAgZWxpZiBkYXRhX3BhdGguaXNfZmlsZSgpOgogICAgICAgIHRleHRfZmlsZXMgPSBbZGF0YV9wYXRoXQogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlVucmVjb2duaXplZCBkYXRhIHBhdGguIFRoZSBwYXJhbWV0ZXIgYGRhdGFfcGF0aGAgbXVzdCBiZSBlaXRoZXIgYSBkaXJlY3RvcnkgcGF0aCBvciBhIGZpbGUgcGF0aC4gIgogICAgICAgICAgICBmIkdpdmVuOiB7c3RyKGRhdGFfcGF0aCl9ICIKICAgICAgICApCgogICAgcmV0dXJuIHRleHRfZmlsZXMKCgpkZWYgX2dldF9wcm9tcHRfdGVtcGxhdGUoCiAgICB0ZXh0X3dyYXBwZXI6IHN0ciwKICAgIHF1ZXN0aW9uc193cmFwcGVyOiBzdHIsCiAgICBxdWVzdGlvbnM6IExpc3Rbc3RyXSwKKSAtPiBzdHI6CgogICAgIyBWYWxpZGF0ZSBhbmQgYnVpbGQgdGhlIHRleHQgd3JhcHBlcjoKICAgIHRleHRfd3JhcHBlciA9IHRleHRfd3JhcHBlciBvciAoCiAgICAgICAgIkdpdmVuIHRoZSBmb2xsb3dpbmcgdGV4dDpcbiIgIi0tLS0tXG4iICJ7fVxuIiAiLS0tLS0iCiAgICApCiAgICBpZiB0ZXh0X3dyYXBwZXIuY291bnQoInt9IikgIT0gMToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAiVGhlIGB0ZXh0X3dyYXBwZXJgIG11c3QgaW5jbHVkZSBvbmUgcGxhY2Vob2xkZXIgJ3t9JyBmb3IgdGhlIHRleHQgb2YgdGhlIGZpbGUgdG8gYmUgYXNrZWQgYWJvdXQuIgogICAgICAgICkKCiAgICAjIFZhbGlkYXRlIGFuZCBidWlsZCB0aGUgcXVlc3Rpb24gd3JhcHBlcjoKICAgIHF1ZXN0aW9uc193cmFwcGVyID0gcXVlc3Rpb25zX3dyYXBwZXIgb3IgIkFuc3dlciB0aGUgcXVlc3Rpb25zOlxuIiAie30iCiAgICBpZiBxdWVzdGlvbnNfd3JhcHBlci5jb3VudCgie30iKSAhPSAxOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICJUaGUgYHF1ZXN0aW9uc193cmFwcGVyYCBtdXN0IGluY2x1ZGUgb25lIHBsYWNlaG9sZGVyICd7fScgZm9yIHRoZSBsaXN0IG9mIHF1ZXN0aW9ucy4iCiAgICAgICAgKQoKICAgICMgVmFsaWRhdGUgYW5kIHBhcnNlIHRoZSBxdWVzdGlvbnM6CiAgICBpZiBsZW4ocXVlc3Rpb25zKSA9PSAwOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIlBsZWFzZSBpbmNsdWRlIGF0IGxlYXN0IG9uZSBxdWVzdGlvbi4iKQogICAgcXVlc3Rpb25zID0gIlxuIi5qb2luKAogICAgICAgIFtmIntpfS4ge3F1ZXN0aW9ufSIgZm9yIGksIHF1ZXN0aW9uIGluIGVudW1lcmF0ZShxdWVzdGlvbnMsIDEpXQogICAgKQoKICAgICMgQ29uc3RydWN0IHRoZSB0ZW1wbGF0ZToKICAgIHJldHVybiBmInt0ZXh0X3dyYXBwZXJ9XG57cXVlc3Rpb25zX3dyYXBwZXIuZm9ybWF0KHF1ZXN0aW9ucyl9XG4iCgoKZGVmIF9nZXRfZ2VuZXJhdGlvbl9waXBlbGluZSgKICAgIG1vZGVsX25hbWU6IHN0ciwKICAgIGRldmljZV9tYXA6IFVuaW9uW3N0ciwgZGljdF0sCiAgICB0b2tlbml6ZXJfbmFtZTogc3RyLAogICAgbW9kZWxfa3dhcmdzOiBkaWN0LAogICAgdG9rZW5pemVyX2t3YXJnczogZGljdCwKICAgIGF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGg6IGludCA9IE5vbmUsCiAgICBiYXRjaF9zaXplOiBpbnQgPSAxLAopOgogICAgIyBMb2FkIHRoZSBtb2RlbDoKICAgIG1vZGVsID0gdHJhbnNmb3JtZXJzLkF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCgKICAgICAgICBtb2RlbF9uYW1lLCBkZXZpY2VfbWFwPWRldmljZV9tYXAsICoqbW9kZWxfa3dhcmdzCiAgICApCgogICAgIyBTZXQgZXhsbGFtYSBtYXggaW5wdXQgbGVuZ3RoIGlmIHByb3ZpZGVkOgogICAgIyBUaGlzIGNoYW5nZXMgdGhlIG1vZGVsJ3MgY29udGV4dCBzaXplLgogICAgaWYgYXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aDoKICAgICAgICBmcm9tIGF1dG9fZ3B0cSBpbXBvcnQgZXhsbGFtYV9zZXRfbWF4X2lucHV0X2xlbmd0aAoKICAgICAgICBtb2RlbCA9IGV4bGxhbWFfc2V0X21heF9pbnB1dF9sZW5ndGgoCiAgICAgICAgICAgIG1vZGVsPW1vZGVsLCBtYXhfaW5wdXRfbGVuZ3RoPWF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGgKICAgICAgICApCgogICAgIyBMb2FkIHRoZSB0b2tlbml6ZXI6CiAgICB0b2tlbml6ZXIgPSB0cmFuc2Zvcm1lcnMuQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoCiAgICAgICAgdG9rZW5pemVyX25hbWUsICoqdG9rZW5pemVyX2t3YXJncwogICAgKQoKICAgICMgSW5pdGlhbGl6ZSBhIGdlbmVyYXRpb24gcGlwbGluZSBhbmQgcmV0dXJuOgogICAgcGlwZSA9IHRyYW5zZm9ybWVycy5waXBlbGluZSgKICAgICAgICB0YXNrPSJ0ZXh0LWdlbmVyYXRpb24iLAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIHRva2VuaXplcj10b2tlbml6ZXIsCiAgICAgICAgYmF0Y2hfc2l6ZT1iYXRjaF9zaXplLAogICAgKQogICAgcGlwZS50b2tlbml6ZXIucGFkX3Rva2VuX2lkID0gbW9kZWwuY29uZmlnLmVvc190b2tlbl9pZAogICAgcmV0dXJuIHBpcGUKCgpkZWYgX3JlYWRfZmlsZV9iYXRjaCgKICAgIGZpbGVfYmF0Y2g6IExpc3RbcGF0aGxpYi5QYXRoXSwKICAgIHByb21wdF90ZW1wbGF0ZTogc3RyLAopIC0+IExpc3Rbc3RyXToKICAgIGJhdGNoID0gW10KCiAgICAjIEdvIG92ZXIgYWxsIGZpbGVzIGFuZCByZWFkIGluIHVzYWJsZSBmb3JtYXQKICAgIGZvciBmaWxlIGluIGZpbGVfYmF0Y2g6CiAgICAgICAgd2l0aCBvcGVuKGZpbGUsICJyIiwgZW5jb2Rpbmc9InV0Zi04IikgYXMgZnA6CiAgICAgICAgICAgIGJhdGNoLmFwcGVuZChwcm9tcHRfdGVtcGxhdGUuZm9ybWF0KGZwLnJlYWQoKSkpCiAgICByZXR1cm4gYmF0Y2gKCgpkZWYgX3RvX2dyb3VwX2xpc3QoYXJndW1lbnRfdmFsdWU6IGxpc3QsIGFyZ3VtZW50X25hbWU6IHN0ciwgbGVuZ3RoOiBpbnQpOgoKICAgICMgQ2hlY2sgaWYgaXMgbGlzdCwgdHVybiB0byBsaXN0IGlmIG5vdAogICAgYXJndW1lbnRfdmFsdWUgPSAoCiAgICAgICAgYXJndW1lbnRfdmFsdWUgaWYgaXNpbnN0YW5jZShhcmd1bWVudF92YWx1ZSwgbGlzdCkgZWxzZSBbYXJndW1lbnRfdmFsdWVdCiAgICApCiAgICBsaXN0X2xlbiA9IGxlbihhcmd1bWVudF92YWx1ZSkKCiAgICAjIElmIG5vdCBhIGxpc3QsIG9yIGlzIGEgbGlzdCBvZiBsZW4gMSB3ZSBkdXBsaWNhdGUgZm9yIGNvcnJlY3QgbGVuZ3RoCiAgICAjIElmIGxpc3QgaW4gd3JvbmcgbGVuZ3RoIHRocm93IGFuIGVycm9yCiAgICBpZiBsaXN0X2xlbiAhPSBsZW5ndGg6CiAgICAgICAgaWYgbGlzdF9sZW4gPT0gMToKICAgICAgICAgICAgcmV0dXJuIGFyZ3VtZW50X3ZhbHVlICogbGVuZ3RoCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJUaGUgYXJndW1lbnQgdmFsdWUgb2YgJ3thcmd1bWVudF9uYW1lfScgaXMgbm90IGVxdWFsIHRvIHRoZSBsZW5ndGggb2YgdGhlIGdpdmVuIHF1ZXN0aW9ucyAtIHtsZW5ndGh9IgogICAgICAgICkKICAgIHJldHVybiBhcmd1bWVudF92YWx1ZQoKCmNsYXNzIFF1ZXN0aW9uSGFuZGxlcjoKICAgICIiIgogICAgQSBjbGFzcyBmb3IgaGFuZGxpbmcgcXVlc3Rpb25zIGFuc3dlcmluZyBmb3IgYSBnaXZlbiBxdWVzdGlvbiB0eXBlLgogICAgVGhpcyBjbGFzcyBpcyB1c2VkIGFzIGEgYmFzZSBjbGFzcyBmb3IgYWxsIHF1ZXN0aW9uIHR5cGVzLCBhbmQgZm9yIGRlZmF1bHQgcXVlc3Rpb24gdHlwZSAocmVndWxhciBxdWVzdGlvbgogICAgYW5zd2VyaW5nIHdpdGhvdXQgYW55IHNwZWNpYWwgaGFuZGxpbmcpLgogICAgIiIiCgogICAgY2xhc3MgQ29uZmlnS2V5czoKICAgICAgICBwYXNzCgogICAgZGVmIF9faW5pdF9fKHNlbGYpOgogICAgICAgIHBhc3MKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgX2dldF9hbnN3ZXJzKGdlbmVyYXRlZF90ZXh0OiBzdHIsIHF1ZXN0aW9uc19hbW91bnQ6IGludCkgLT4gTGlzdFtzdHJdOgoKICAgICAgICAjIENsZWFyIGFuc3dlciBzdGFydCAocGFydCBiZWZvcmUgbnVtYmVycyk6CiAgICAgICAgIyBUT0RPIGZpbmQgYmV0dGVyIHdheSB0byB2ZXJpZnksIGZvciBsaXN0IG9mIHF1ZXN0aW9ucyB0aGlzIGlzIHJlZHVuZGFudCBmb3IgZXhhbXBsZQogICAgICAgIGlmICIxLiIgbm90IGluIGdlbmVyYXRlZF90ZXh0OgogICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgZiJBbnN3ZXIgMS4gaXMgbWlzc2luZyBmcm9tIHRoZSBnZW5lcmF0ZWQgdGV4dDogJ3tnZW5lcmF0ZWRfdGV4dH0nIgogICAgICAgICAgICApCiAgICAgICAgdGV4dCA9IGdlbmVyYXRlZF90ZXh0LnNwbGl0KCIxLiIsIDEpWzFdCgogICAgICAgICMgU3RhcnQgZXh0cmFjdGluZyB0aGUgYW5zd2VyczoKICAgICAgICBhbnN3ZXJzID0gW10KICAgICAgICBmb3IgaSBpbiByYW5nZSgxLCBxdWVzdGlvbnNfYW1vdW50ICsgMSk6CiAgICAgICAgICAgICMgSWYgaXQncyB0aGUgbGFzdCBhbnN3ZXIgdG8gbG9vayBmb3IsIHRha2UgdGhlIHJlc3Qgb2YgdGhlIHRleHQ6CiAgICAgICAgICAgIGlmIGkgPT0gcXVlc3Rpb25zX2Ftb3VudDoKICAgICAgICAgICAgICAgIGFuc3dlcl9pID0gdGV4dAogICAgICAgICAgICAjIFZlcmlmeSB0aGVyZSBpcyBhIHF1ZXN0aW9uIG51bWJlciBpbiB0aGUgdGV4dDoKICAgICAgICAgICAgZWxpZiBmIntpICsgMX0uIiBub3QgaW4gdGV4dDoKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgZiJBbnN3ZXIge2kgKyAxfS4gaXMgbWlzc2luZyBmcm9tIHRoZSBnZW5lcmF0ZWQgdGV4dDogJ3tnZW5lcmF0ZWRfdGV4dH0nIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAjIFRha2UgaSdzIGFuc3dlcjoKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGFuc3dlcl9pLCB0ZXh0ID0gdGV4dC5zcGxpdChmIntpICsgMX0uIiwgMSkKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSBhbnN3ZXIgcmVtb3ZpbmcgcmVkdW5kYW50IHNwYWNlczoKICAgICAgICAgICAgYW5zd2Vycy5hcHBlbmQoYW5zd2VyX2kuc3RyaXAoKSkKCiAgICAgICAgcmV0dXJuIGFuc3dlcnMKCiAgICBkZWYgX2luZmVyX3F1ZXN0aW9ucygKICAgICAgICBzZWxmLAogICAgICAgIHF1ZXN0aW9uc19hbW91bnQ6IGludCwKICAgICAgICBiYXRjaGVkX2lucHV0OiBMaXN0W3N0cl0sCiAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZTogdHJhbnNmb3JtZXJzLlBpcGVsaW5lLAogICAgICAgIGdlbmVyYXRpb25fY29uZmlnOiB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZywKICAgICkgLT4gTGlzdFtMaXN0W3N0cl1dOgoKICAgICAgICAjIEluZmVyIHRocm91Z2ggdGhlIGxsbToKICAgICAgICBiYXRjaGVkX291dHB1dCA9IGdlbmVyYXRpb25fcGlwZWxpbmUoCiAgICAgICAgICAgIGJhdGNoZWRfaW5wdXQsCiAgICAgICAgICAgIGdlbmVyYXRpb25fY29uZmlnPWdlbmVyYXRpb25fY29uZmlnLAogICAgICAgICAgICBlb3NfdG9rZW5faWQ9Z2VuZXJhdGlvbl9waXBlbGluZS50b2tlbml6ZXIuZW9zX3Rva2VuX2lkLAogICAgICAgICAgICByZXR1cm5fZnVsbF90ZXh0PUZhbHNlLAogICAgICAgICAgICBudW1fcmV0dXJuX3NlcXVlbmNlcz0xLAogICAgICAgICkKCiAgICAgICAgIyBQcm9jZXNzIHRoZSBvdXRwdXRzIHRvIGdldCB0aGUgYW5zd2VyczoKICAgICAgICBiYXRjaGVkX2Fuc3dlcnMgPSBbXQogICAgICAgIGZvciBvdXRwdXQgaW4gYmF0Y2hlZF9vdXRwdXQ6CiAgICAgICAgICAgICMgR2V0IHRoZSBnZW5lcmF0ZWQgYW5zd2VyczoKICAgICAgICAgICAgYW5zd2VycyA9IHNlbGYuX2dldF9hbnN3ZXJzKAogICAgICAgICAgICAgICAgZ2VuZXJhdGVkX3RleHQ9b3V0cHV0WzBdWyJnZW5lcmF0ZWRfdGV4dCJdLAogICAgICAgICAgICAgICAgcXVlc3Rpb25zX2Ftb3VudD1xdWVzdGlvbnNfYW1vdW50LAogICAgICAgICAgICApCiAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgcHJvY2Vzc2VkIGFuc3dlcnM6CiAgICAgICAgICAgIGJhdGNoZWRfYW5zd2Vycy5hcHBlbmQoYW5zd2VycykKICAgICAgICByZXR1cm4gYmF0Y2hlZF9hbnN3ZXJzCgogICAgZGVmIGFuc3dlcigKICAgICAgICBzZWxmLAogICAgICAgIHF1ZXN0aW9uc19hbW91bnQ6IGludCwKICAgICAgICBiYXRjaGVkX2lucHV0OiBMaXN0W3N0cl0sCiAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZTogdHJhbnNmb3JtZXJzLlBpcGVsaW5lLAogICAgICAgIGdlbmVyYXRpb25fY29uZmlnOiB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZywKICAgICkgLT4gTGlzdFtMaXN0W3N0cl1dOgogICAgICAgICIiIgogICAgICAgIEFuc3dlciBxdWVzdGlvbnMgd2l0aCBhIGNvbnRleHQgdG8gdGhlIGdpdmVuIHRleHQgZmlsZXMgY29udGVudHMgYnkgYSBwcmV0cmFpbmVkIExMTSBtb2RlbCBpbiBnaXZlbiBwaXBlbGluZS4KICAgICAgICAiIiIKICAgICAgICByZXR1cm4gc2VsZi5faW5mZXJfcXVlc3Rpb25zKAogICAgICAgICAgICBxdWVzdGlvbnNfYW1vdW50PXF1ZXN0aW9uc19hbW91bnQsCiAgICAgICAgICAgIGJhdGNoZWRfaW5wdXQ9YmF0Y2hlZF9pbnB1dCwKICAgICAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZT1nZW5lcmF0aW9uX3BpcGVsaW5lLAogICAgICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZz1nZW5lcmF0aW9uX2NvbmZpZywKICAgICAgICApCgoKY2xhc3MgUG9sbFF1ZXN0aW9uSGFuZGxlcihRdWVzdGlvbkhhbmRsZXIpOgogICAgIiIiCiAgICBTdGF0aWMgY2xhc3MgdG8gaG9sZCBhbGwgdGhlIHBvc3NpYmxlIHBvbGwgcXVlc3Rpb24gY29uZmlndXJhdGlvbnMgb3B0aW9ucyBrZXlzCiAgICAiIiIKCiAgICBjbGFzcyBDb25maWdLZXlzOgogICAgICAgICIiIgogICAgICAgIEEgY2xhc3MgZm9yIGhhbmRsaW5nIHF1ZXN0aW9ucyBhbnN3ZXJpbmcgZm9yIHBvbGwgdHlwZSBxdWVzdGlvbnMuCiAgICAgICAgVGhlc2UgdHlwZSBvZiBxdWVzdGlvbiBhcmUgYW5zd2VyZWQgYnkgYXNraW5nIHRoZSBzYW1lIHF1ZXN0aW9uIG11bHRpcGxlIHRpbWVzCiAgICAgICAgYW5kIGNob29zaW5nIHRoZSBtb3N0IGNvbW1vbiBhbnN3ZXIgb3IgdGhlIGF2ZXJhZ2UgYW5zd2VyLgogICAgICAgICIiIgoKICAgICAgICAjOiBUaGUgbnVtYmVyIG9mIHRpbWVzIHRvIGFzayB0aGUgc2FtZSBxdWVzdGlvbi4KICAgICAgICBQT0xMX0NPVU5UID0gInBvbGxfY291bnQiCgogICAgICAgICM6IFRoZSBzdHJhdGVneSB0byB1c2UgZm9yIGNob29zaW5nIHRoZSBhbnN3ZXIgZnJvbSB0aGUgcG9sbC4KICAgICAgICBQT0xMX1NUUkFURUdZID0gInBvbGxfc3RyYXRlZ3kiCgogICAgY2xhc3MgU3RyYXRlZ3koZW51bS5FbnVtKToKICAgICAgICAjOiBUaGUgbW9zdCBjb21tb24gYW5zd2VyIHN0cmF0ZWd5LgogICAgICAgIE1PU1RfQ09NTU9OID0gIm1vc3RfY29tbW9uIgoKICAgICAgICAjOiBUaGUgYXZlcmFnZSBhbnN3ZXIgc3RyYXRlZ3kuCiAgICAgICAgQVZFUkFHRSA9ICJhdmVyYWdlIgoKICAgICAgICBAc3RhdGljbWV0aG9kCiAgICAgICAgZGVmIG1vc3RfY29tbW9uKGFuc3dlcnMpOgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgQ2FsY3VsYXRlIHRoZSBtb3N0IGNvbW1vbiBhbnN3ZXIgZm9yIGEgZ2l2ZW4gbGlzdCBvZiBhbnN3ZXJzLgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgY291bnQgPSBDb3VudGVyKGFuc3dlcnMpCiAgICAgICAgICAgIG1vc3RfY29tbW9uID0gY291bnQubW9zdF9jb21tb24oMSkKICAgICAgICAgICAgcmV0dXJuIG1vc3RfY29tbW9uWzBdWzBdCgogICAgICAgIEBzdGF0aWNtZXRob2QKICAgICAgICBkZWYgYXZlcmFnZShhbnN3ZXJzKToKICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIENhbGN1bGF0ZSB0aGUgYXZlcmFnZSBhbnN3ZXIgZm9yIGEgZ2l2ZW4gbGlzdCBvZiBhbnN3ZXJzLgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShhbnN3ZXJzWzBdLCBzdHIpOgogICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAiQ2Fubm90IHBlcmZvcm0gcG9sbCB3aXRoIGF2ZXJhZ2UgYW5zd2VyIHN0cmF0ZWd5IG9mIG5vbiBudW1lcmljIHZhbHVlcywiCiAgICAgICAgICAgICAgICAgICAgIiBwbGVhc2UgY2hhbmdlIHRoZSBxdWVzdGlvbiB0byBnaXZlIG51bWVyaWMgZGF0YSwgb3IgY2hvb3NlICdtb3N0X2NvbW1vbicgYXMgc3RyYXRlZ3kuIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgbnVtZXJpY192YWx1ZXMgPSBhbnN3ZXJzCiAgICAgICAgICAgIGF2ZyA9IHN1bShudW1lcmljX3ZhbHVlcykgLyBsZW4obnVtZXJpY192YWx1ZXMpCgogICAgICAgICAgICAjIFJvdW5kIHRvIHRoZSBjbG9zZXN0IGludGVnZXIgYW5kIHJldHVybiBjb3JyZXNwb25kaW5nIHZhbHVlCiAgICAgICAgICAgIHJldHVybiByb3VuZChhdmcpCgogICAgICAgIGRlZiBkbyhzZWxmLCBhbnN3ZXJzKToKICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIFBlcmZvcm0gdGhlIHN0cmF0ZWd5LgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgcmV0dXJuIGdldGF0dHIoc2VsZiwgc2VsZi52YWx1ZSkoYW5zd2VycykKCiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwgcG9sbF9jb3VudDogaW50ID0gNSwgcG9sbF9zdHJhdGVneTogc3RyID0gIm1vc3RfY29tbW9uIik6CiAgICAgICAgc3VwZXIoKS5fX2luaXRfXygpCiAgICAgICAgc2VsZi5wb2xsX2NvdW50ID0gcG9sbF9jb3VudAogICAgICAgIHNlbGYucG9sbF9zdHJhdGVneSA9IHNlbGYuU3RyYXRlZ3kocG9sbF9zdHJhdGVneSkKCiAgICBkZWYgYW5zd2VyKAogICAgICAgIHNlbGYsCiAgICAgICAgcXVlc3Rpb25zX2Ftb3VudDogaW50LAogICAgICAgIGJhdGNoZWRfaW5wdXQ6IExpc3Rbc3RyXSwKICAgICAgICBnZW5lcmF0aW9uX3BpcGVsaW5lOiB0cmFuc2Zvcm1lcnMuUGlwZWxpbmUsCiAgICAgICAgZ2VuZXJhdGlvbl9jb25maWc6IHRyYW5zZm9ybWVycy5HZW5lcmF0aW9uQ29uZmlnLAogICAgKSAtPiBMaXN0W0xpc3Rbc3RyXV06CiAgICAgICAgIiIiCiAgICAgICAgQW5zd2VyIHF1ZXN0aW9ucyB3aXRoIGEgY29udGV4dCB0byB0aGUgZ2l2ZW4gdGV4dCBmaWxlcyBjb250ZW50cyBieSBhIHByZXRyYWluZWQgTExNIG1vZGVsIGluIGdpdmVuIHBpcGVsaW5lLgogICAgICAgICIiIgogICAgICAgIHJldHVybiBzZWxmLl9hbnN3ZXJfcG9sbF9xdWVzdGlvbnMoCiAgICAgICAgICAgIHF1ZXN0aW9uc19hbW91bnQ9cXVlc3Rpb25zX2Ftb3VudCwKICAgICAgICAgICAgYmF0Y2hlZF9pbnB1dD1iYXRjaGVkX2lucHV0LAogICAgICAgICAgICBnZW5lcmF0aW9uX3BpcGVsaW5lPWdlbmVyYXRpb25fcGlwZWxpbmUsCiAgICAgICAgICAgIGdlbmVyYXRpb25fY29uZmlnPWdlbmVyYXRpb25fY29uZmlnLAogICAgICAgICkKCiAgICBkZWYgX2Fuc3dlcl9wb2xsX3F1ZXN0aW9ucygKICAgICAgICBzZWxmLAogICAgICAgIHF1ZXN0aW9uc19hbW91bnQ6IGludCwKICAgICAgICBiYXRjaGVkX2lucHV0OiBMaXN0W3N0cl0sCiAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZTogdHJhbnNmb3JtZXJzLlBpcGVsaW5lLAogICAgICAgIGdlbmVyYXRpb25fY29uZmlnOiB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZywKICAgICkgLT4gTGlzdFtMaXN0W3N0cl1dOgogICAgICAgIHZvdGVzID0gW10KCiAgICAgICAgIyBSdW4gdGhlIHBvbGwgZm9yIGVhY2ggcXVlc3Rpb24KICAgICAgICBmb3IgXyBpbiByYW5nZShzZWxmLnBvbGxfY291bnQpOgogICAgICAgICAgICBiYXRjaGVkX2Fuc3dlcnMgPSBzZWxmLl9pbmZlcl9xdWVzdGlvbnMoCiAgICAgICAgICAgICAgICBxdWVzdGlvbnNfYW1vdW50PXF1ZXN0aW9uc19hbW91bnQsCiAgICAgICAgICAgICAgICBiYXRjaGVkX2lucHV0PWJhdGNoZWRfaW5wdXQsCiAgICAgICAgICAgICAgICBnZW5lcmF0aW9uX3BpcGVsaW5lPWdlbmVyYXRpb25fcGlwZWxpbmUsCiAgICAgICAgICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZz1nZW5lcmF0aW9uX2NvbmZpZywKICAgICAgICAgICAgKQogICAgICAgICAgICB2b3Rlcy5hcHBlbmQoYmF0Y2hlZF9hbnN3ZXJzKQogICAgICAgIGFuc3dlcnMgPSBbXQoKICAgICAgICAjIENvbGxlY3QgdGhlIGFuc3dlcnMgYWNjb3JkaW5nIHRvIHRoZSBwb2xsIHN0cmF0ZWd5CiAgICAgICAgIyBBdmVyYWdlIHN0cmF0ZWd5IHdvcmtzIGZvciBudW1lcmljIHZhbHVlcyBvbmx5CiAgICAgICAgZm9yIGJhdGNoIGluIHJhbmdlKGxlbih2b3Rlc1swXSkpOgogICAgICAgICAgICBiYXRjaGVkX2Fuc3dlcnMgPSBbXQogICAgICAgICAgICBmb3IgcXVlc3Rpb24gaW4gcmFuZ2UocXVlc3Rpb25zX2Ftb3VudCk6CiAgICAgICAgICAgICAgICAjIENyZWF0ZSBhIGxpc3Qgb2YgYWxsIGFuc3dlcnMgdG8gcmVsZXZhbnQgcXVlc3Rpb24KICAgICAgICAgICAgICAgIGFuc3dlciA9IFsKICAgICAgICAgICAgICAgICAgICB2b3Rlc1t2b3Rlcl1bYmF0Y2hdW3F1ZXN0aW9uXSBmb3Igdm90ZXIgaW4gcmFuZ2Uoc2VsZi5wb2xsX2NvdW50KQogICAgICAgICAgICAgICAgXQogICAgICAgICAgICAgICAgYW5zd2VyID0gc2VsZi5wb2xsX3N0cmF0ZWd5LmRvKGFuc3dlcikKICAgICAgICAgICAgICAgIGJhdGNoZWRfYW5zd2Vycy5hcHBlbmQoYW5zd2VyKQogICAgICAgICAgICBhbnN3ZXJzLmFwcGVuZChiYXRjaGVkX2Fuc3dlcnMpCiAgICAgICAgcmV0dXJuIGFuc3dlcnMKCgojIEhvbGRzIG5hbWVzIG9mIFF1ZXN0aW9uSGFuZGxlcwpjbGFzcyBRdWVzdGlvblR5cGVzOgogICAgREVGQVVMVCA9ICJkZWZhdWx0IgogICAgUE9MTCA9ICJwb2xsIgoKCiMgTWFwcyBxdWVzdGlvbiB0eXBlcyB0byB0aGVpciBoYW5kbGVycwpRVUVTVElPTl9NQVBQSU5HID0gewogICAgUXVlc3Rpb25UeXBlcy5ERUZBVUxUOiBRdWVzdGlvbkhhbmRsZXIsCiAgICBRdWVzdGlvblR5cGVzLlBPTEw6IFBvbGxRdWVzdGlvbkhhbmRsZXIsCn0K base_image: mlrun/mlrun commands: [] code_origin: '' origin_filename: '' requirements: - - transformers torch tqdm + - transformers + - torch + - tqdm entry_points: open_mpi_handler: name: open_mpi_handler @@ -27,29 +29,30 @@ spec: parameters: - name: worker_inputs type: List[str] - default: '' - name: root_worker_inputs type: Dict[str, Any] default: null - outputs: - - default: '' + outputs: [] lineno: 58 + has_varargs: false + has_kwargs: false decorator: name: decorator doc: '' parameters: - name: handler - default: '' - outputs: - - default: '' + outputs: [] lineno: 66 + has_varargs: false + has_kwargs: false wrapper: name: wrapper doc: '' parameters: [] - outputs: - - default: '' + outputs: [] lineno: 71 + has_varargs: false + has_kwargs: true answer_questions: name: answer_questions doc: 'Answer questions with a context to the given text files contents by a @@ -81,19 +84,16 @@ spec: type: Union[str, List[str]] doc: A path to a directory of text files or a path to a text file to ask questions about. - default: '' - name: model_name type: str doc: The pre-trained model name from the huggingface hub to use for asking questions. - default: '' - name: questions type: Union[List[str], List[List[str]]] doc: The questions to ask. A list of lists of questions to ask per text file, and devided by question groups, the groups can be dtermained by size (in order to avoid large inputs to the llm) or by questioning method (regular or poll like questioning). - default: '' - name: device_map type: Union[str, dict] doc: A map to use for loading the model on multiple devices. @@ -152,60 +152,58 @@ spec: doc: 'Whether to present logs of a progress bar and errors. Default: True.' default: false outputs: - - default: '' - doc: 'A tuple of:' + - doc: 'A tuple of:' + type: Tuple[pd.DataFrame, dict] lineno: 130 + has_varargs: false + has_kwargs: false answer: name: answer doc: Answer questions with a context to the given text files contents by a pretrained LLM model in given pipeline. parameters: - name: self - default: '' - name: questions_amount type: int - default: '' - name: batched_input type: List[str] - default: '' - name: generation_pipeline type: Pipeline - default: '' - name: generation_config type: GenerationConfig - default: '' outputs: - - default: '' + - type: List[List[str]] lineno: 674 + has_varargs: false + has_kwargs: false most_common: name: most_common doc: Calculate the most common answer for a given list of answers. parameters: - name: answers - default: '' - outputs: - - default: '' + outputs: [] lineno: 637 + has_varargs: false + has_kwargs: false average: name: average doc: Calculate the average answer for a given list of answers. parameters: - name: answers - default: '' - outputs: - - default: '' + outputs: [] lineno: 646 + has_varargs: false + has_kwargs: false do: name: do doc: Perform the strategy. parameters: - name: self - default: '' - name: answers - default: '' - outputs: - - default: '' + outputs: [] lineno: 662 + has_varargs: false + has_kwargs: false description: GenAI approach of question answering on a given data default_handler: answer_questions disable_auto_mount: false diff --git a/question_answering/item.yaml b/question_answering/item.yaml index 6daa1b564..58ab5cc36 100755 --- a/question_answering/item.yaml +++ b/question_answering/item.yaml @@ -20,8 +20,8 @@ spec: image: mlrun/mlrun kind: job requirements: - transformers - torch - tqdm + - transformers + - torch + - tqdm url: '' -version: 0.3.0 +version: 0.3.1 diff --git a/requirements.txt b/requirements.txt index be36c8c86..faa20126f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,9 @@ black~=22.0 isort~=5.7 sphinx==4.0.2 sphinx-book-theme==0.3.3 -sphinx-togglebutton==0.3.1 \ No newline at end of file +sphinx-togglebutton==0.3.1 +sphinxcontrib-applehelp<=1.0.7 +sphinxcontrib.devhelp<=1.0.5 +sphinxcontrib-htmlhelp<=2.0.4 +sphinxcontrib-serializinghtml<=1.1.9 +sphinxcontrib-qthelp<=1.0.6 \ No newline at end of file diff --git a/silero_vad/assets/test_data.wav b/silero_vad/assets/test_data.wav new file mode 100644 index 000000000..a3a993c20 Binary files /dev/null and b/silero_vad/assets/test_data.wav differ diff --git a/silero_vad/function.yaml b/silero_vad/function.yaml new file mode 100644 index 000000000..75d1ce0cc --- /dev/null +++ b/silero_vad/function.yaml @@ -0,0 +1,280 @@ +kind: job +metadata: + name: silero-vad + tag: '' + hash: bc0ad5572cc391fcdc93baaee48e1ef949a7984d + project: '' + labels: + author: guyl + categories: + - Deep Learning + - PyTorch + - Audio +spec: + command: '' + args: [] + image: '' + build: + functionSourceCode: IyBDb3B5cmlnaHQgMjAyNCBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgbG9nZ2luZwpmcm9tIG11bHRpcHJvY2Vzc2luZyBpbXBvcnQgUHJvY2VzcywgUXVldWUKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwZXMgaW1wb3J0IEZ1bmN0aW9uVHlwZQpmcm9tIHR5cGluZyBpbXBvcnQgRGljdCwgTGlzdCwgVHVwbGUsIFR5cGUsIFVuaW9uCgppbXBvcnQgdG9yY2gKaW1wb3J0IHRvcmNoYXVkaW8KZnJvbSB0cWRtIGltcG9ydCB0cWRtCgoKY2xhc3MgQmFzZVRhc2s6CiAgICAiIiIKICAgIEEgYmFzZSBjbGFzcyBmb3IgYSB0YXNrIHRvIGNvbXBsZXRlIGFmdGVyIFZBRC4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCBhdWRpb19maWxlOiBQYXRoKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSBiYXNlIHRhc2suCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlOiBUaGUgYXVkaW8gZmlsZSBhc3NpZ25lZCB0byB0aGUgdGFzay4KICAgICAgICAiIiIKICAgICAgICAjIFN0b3JlIHRoZSBhdWRpbyBmaWxlOgogICAgICAgIHNlbGYuX2F1ZGlvX2ZpbGUgPSBhdWRpb19maWxlCgogICAgICAgICMgUHJlcGFyZSB0aGUgcmVzdWx0OgogICAgICAgIHNlbGYuX3Jlc3VsdCA9IE5vbmUKCiAgICBAcHJvcGVydHkKICAgIGRlZiBhdWRpb19maWxlKHNlbGYpIC0+IFBhdGg6CiAgICAgICAgIiIiCiAgICAgICAgR2V0IHRoZSBhdWRpbyBmaWxlIG9mIHRoZSB0YXNrLgoKICAgICAgICA6cmV0dXJuczogVGhlIGF1ZGlvIGZpbGUgb2YgdGhlIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX2F1ZGlvX2ZpbGUKCiAgICBkZWYgZG9fdGFzaygKICAgICAgICBzZWxmLCBzcGVlY2hfdGltZXN0YW1wczogVW5pb25bTGlzdFtEaWN0W3N0ciwgaW50XV0sIExpc3RbTGlzdFtEaWN0W3N0ciwgaW50XV1dXQogICAgKToKICAgICAgICAiIiIKICAgICAgICBEbyB0aGUgdGFzayBvbiB0aGUgZ2l2ZW4gc3BlZWNoIHRpbWVzdGFtcHMuIFRoZSBiYXNlIHRhc2sgd2lsbCBzaW1wbHkgc2F2ZSB0aGUgc3BlZWNoIHRpbWVzdGFtcHMgYXMgdGhlIHJlc3VsdC4KCiAgICAgICAgOnBhcmFtIHNwZWVjaF90aW1lc3RhbXBzOiBUaGUgc3BlZWNoIHRpbWVzdGFtcHMgdG8gZG8gdGhlIHRhc2sgb24gYXMgb3V0cHV0dGVkIGZyb20gdGhlIFZBRC4KICAgICAgICAiIiIKICAgICAgICBzZWxmLl9yZXN1bHQgPSBzcGVlY2hfdGltZXN0YW1wcwoKICAgIGRlZiBnZXRfcmVzdWx0KHNlbGYpIC0+IFR1cGxlW3N0ciwgbGlzdF06CiAgICAgICAgIiIiCiAgICAgICAgR2V0IHRoZSByZXN1bHQgb2YgdGhlIHRhc2suIEEgdHVwbGUgb2YgdGhlIGF1ZGlvIGZpbGUgbmFtZSBhbmQgdGhlIHJlc3VsdC4KCiAgICAgICAgOnJldHVybnM6IFRoZSByZXN1bHQgb2YgdGhlIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX2F1ZGlvX2ZpbGUubmFtZSwgc2VsZi5fcmVzdWx0CgogICAgZGVmIHRvX3R1cGxlKHNlbGYpIC0+IFR1cGxlW3N0ciwgZGljdF06CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCB0aGUgdGFzayB0byBhIHR1cGxlIHRvIHJlY29uc3RydWN0IGl0IGxhdGVyICh1c2VkIGZvciBtdWx0aXByb2Nlc3NpbmcgdG8gcGFzcyBpbiBxdWV1ZSkuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgY29udmVydGVkIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX19jbGFzc19fLl9fbmFtZV9fLCB7ImF1ZGlvX2ZpbGUiOiBzZWxmLl9hdWRpb19maWxlfQoKCmNsYXNzIFNwZWVjaERpYXJpemF0aW9uVGFzayhCYXNlVGFzayk6CiAgICAiIiIKICAgIEEgc3BlZWNoIGRpYXJpemF0aW9uIHRhc2suIFRoZSB0YXNrIHdpbGwgZGlhcml6ZSB0aGUgVkFEIHNwZWVjaCB0aW1lc3RhbXBzIGludG8gc3BlYWtlcnMuCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oc2VsZiwgYXVkaW9fZmlsZTogUGF0aCwgc3BlYWtlcl9sYWJlbHM6IExpc3Rbc3RyXSk6CiAgICAgICAgIiIiCiAgICAgICAgSW5pdGlhbGl6ZSB0aGUgc3BlZWNoIGRpYXJpemF0aW9uIHRhc2suCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlOiAgICAgVGhlIGF1ZGlvIGZpbGUgYXNzaWduZWQgdG8gdGhlIHRhc2suCiAgICAgICAgOnBhcmFtIHNwZWFrZXJfbGFiZWxzOiBUaGUgc3BlYWtlciBsYWJlbHMgdG8gdXNlIGZvciB0aGUgZGlhcml6YXRpb24uIElmIG5vdCBnaXZlbiwgdGhlIHNwZWFrZXJzIHdpbGwgYmUgbmFtZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzcGVha2VyXzAiLCAic3BlYWtlcl8xIiwgZXRjLgogICAgICAgICIiIgogICAgICAgIHN1cGVyKCkuX19pbml0X18oYXVkaW9fZmlsZT1hdWRpb19maWxlKQogICAgICAgIHNlbGYuX3NwZWFrZXJfbGFiZWxzID0gc3BlYWtlcl9sYWJlbHMKCiAgICBkZWYgZG9fdGFzayhzZWxmLCBzcGVlY2hfdGltZXN0YW1wczogTGlzdFtMaXN0W0RpY3Rbc3RyLCBpbnRdXV0pOgogICAgICAgICIiIgogICAgICAgIERvIHRoZSB0YXNrIG9uIHRoZSBnaXZlbiBzcGVlY2ggdGltZXN0YW1wcy4gVGhlIHRhc2sgd2lsbCBkaWFyaXplIHRoZSBWQUQgc3BlZWNoIHRpbWVzdGFtcHMgaW50byBzcGVha2Vycy4KCiAgICAgICAgOnBhcmFtIHNwZWVjaF90aW1lc3RhbXBzOiBUaGUgc3BlZWNoIHRpbWVzdGFtcHMgcGVyIGNoYW5uZWwgdG8gZG8gdGhlIHRhc2sgb24gYXMgb3V0cHV0dGVkIGZyb20gdGhlIFZBRC4KICAgICAgICAiIiIKICAgICAgICAjIEdldCB0aGUgc3BlYWtlciBsYWJlbHMgKHNldCBkZWZhdWx0IGlmIG5vdCBnaXZlbik6CiAgICAgICAgc3BlYWtlcl9sYWJlbHMgPSBzZWxmLl9zcGVha2VyX2xhYmVscyBvciBbCiAgICAgICAgICAgIGYic3BlYWtlcl97aX0iIGZvciBpIGluIHJhbmdlKGxlbihzcGVlY2hfdGltZXN0YW1wcykpCiAgICAgICAgXQoKICAgICAgICAjIERpYXJpemUgLSBvcmdhbml6ZSB0aGUgc3BlZWNoIHRpbWVzdGFtcHMgaW50byBhIHNpbmdsZSBsaXN0IG9mIHNwZWFrZXJzIGFuZCBzb3J0IGl0IGJ5IHN0YXJ0IHRpbWU6CiAgICAgICAgc3BlZWNoX2RpYXJpemF0aW9uID0gWwogICAgICAgICAgICAoc3BlZWNoX3RpbWVzdGFtcFsic3RhcnQiXSwgc3BlZWNoX3RpbWVzdGFtcFsiZW5kIl0sIHNwZWFrZXJfbGFiZWwpCiAgICAgICAgICAgIGZvciBzcGVha2VyX2xhYmVsLCBjaGFubmVsX3NwZWVjaF90aW1lc3RhbXBzIGluIHppcCgKICAgICAgICAgICAgICAgIHNwZWFrZXJfbGFiZWxzLCBzcGVlY2hfdGltZXN0YW1wcwogICAgICAgICAgICApCiAgICAgICAgICAgIGZvciBzcGVlY2hfdGltZXN0YW1wIGluIGNoYW5uZWxfc3BlZWNoX3RpbWVzdGFtcHMKICAgICAgICBdCiAgICAgICAgc3BlZWNoX2RpYXJpemF0aW9uLnNvcnQoKQogICAgICAgIHNlbGYuX3Jlc3VsdCA9IHNwZWVjaF9kaWFyaXphdGlvbgoKICAgIGRlZiB0b190dXBsZShzZWxmKSAtPiBUdXBsZVtzdHIsIGRpY3RdOgogICAgICAgICIiIgogICAgICAgIENvbnZlcnQgdGhlIHRhc2sgdG8gYSB0dXBsZSB0byByZWNvbnN0cnVjdCBpdCBsYXRlciAodXNlZCBmb3IgbXVsdGlwcm9jZXNzaW5nIHRvIHBhc3MgaW4gcXVldWUpLgoKICAgICAgICA6cmV0dXJuczogVGhlIGNvbnZlcnRlZCB0YXNrLgogICAgICAgICIiIgogICAgICAgIHRhc2tfY2xhc3MsIHRhc2tfa3dhcmdzID0gc3VwZXIoKS50b190dXBsZSgpCiAgICAgICAgcmV0dXJuIHRhc2tfY2xhc3MsIHsqKnRhc2tfa3dhcmdzLCAic3BlYWtlcl9sYWJlbHMiOiBzZWxmLl9zcGVha2VyX2xhYmVsc30KCgpjbGFzcyBUYXNrQ3JlYXRvcjoKICAgICIiIgogICAgQSB0YXNrIGNyZWF0b3IgdG8gY3JlYXRlIGRpZmZlcmVudCB0YXNrcyB0byBydW4gYWZ0ZXIgdGhlIFZBRC4KICAgICIiIgoKICAgICM6IEEgbWFwIGZyb20gdGFzayBjbGFzcyBuYW1lIHRvIHRhc2sgY2xhc3MgdG8gdXNlIGluIGBmcm9tX3R1cGxlYDoKICAgIF9NQVAgPSB7CiAgICAgICAgQmFzZVRhc2suX19uYW1lX186IEJhc2VUYXNrLAogICAgICAgIFNwZWVjaERpYXJpemF0aW9uVGFzay5fX25hbWVfXzogU3BlZWNoRGlhcml6YXRpb25UYXNrLAogICAgfQoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCB0YXNrX3R5cGU6IFR5cGVbQmFzZVRhc2tdLCB0YXNrX2t3YXJnczogZGljdCA9IE5vbmUpOgogICAgICAgICIiIgogICAgICAgIEluaXRpYWxpemUgdGhlIHRhc2sgY3JlYXRvci4KICAgICAgICA6cGFyYW0gdGFza190eXBlOiBUaGUgdGFzayB0eXBlIC0gYSBgQmFzZVRhc2tgIHN1YmNsYXNzLgogICAgICAgIDpwYXJhbSB0YXNrX2t3YXJnczogQWRkaXRpb25hbCBrZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIHRvIHRoZSB0byBiZSBjcmVhdGVkIHRhc2tzLgogICAgICAgICIiIgogICAgICAgIHNlbGYuX3Rhc2tfdHlwZSA9IHRhc2tfdHlwZQogICAgICAgIHNlbGYuX3Rhc2tfa3dhcmdzID0gdGFza19rd2FyZ3Mgb3Ige30KCiAgICBkZWYgY3JlYXRlX3Rhc2soc2VsZiwgYXVkaW9fZmlsZTogUGF0aCkgLT4gQmFzZVRhc2s6CiAgICAgICAgIiIiCiAgICAgICAgQ3JlYXRlIGEgdGFzayB3aXRoIHRoZSBnaXZlbiBhdWRpbyBmaWxlLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogVGhlIGF1ZGlvIGZpbGUgdG8gYXNzaWduIHRvIHRoZSB0YXNrLgoKICAgICAgICA6cmV0dXJuczogVGhlIGNyZWF0ZWQgdGFzay4KICAgICAgICAiIiIKICAgICAgICByZXR1cm4gc2VsZi5fdGFza190eXBlKGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSwgKipzZWxmLl90YXNrX2t3YXJncykKCiAgICBAY2xhc3NtZXRob2QKICAgIGRlZiBmcm9tX3R1cGxlKGNscywgdGFza190dXBsZTogVHVwbGVbc3RyLCBkaWN0XSkgLT4gQmFzZVRhc2s6CiAgICAgICAgIiIiCiAgICAgICAgQ3JlYXRlIGEgdGFzayBmcm9tIGEgdHVwbGUgb2YgdGhlIGF1ZGlvIGZpbGUgbmFtZSBhbmQgdGhlIHRhc2sga3dhcmdzLgoKICAgICAgICA6cGFyYW0gdGFza190dXBsZTogVGhlIHRhc2sgdHVwbGUgdG8gY3JlYXRlIHRoZSB0YXNrIGZyb20uCgogICAgICAgIDpyZXR1cm5zOiBUaGUgY3JlYXRlZCB0YXNrLgogICAgICAgICIiIgogICAgICAgIHRhc2tfY2xhc3MsIHRhc2tfa3dhcmdzID0gdGFza190dXBsZQogICAgICAgIHJldHVybiBjbHMuX01BUFt0YXNrX2NsYXNzXSgqKnRhc2tfa3dhcmdzKQoKCmNsYXNzIFZvaWNlQWN0aXZpdHlEZXRlY3RvcjoKICAgICIiIgogICAgQSB2b2ljZSBhY3Rpdml0eSBkZXRlY3Rpb24gd3JhcHBlciBmb3IgdGhlIHNpbGVybyBWQUQgbW9kZWwgLSBodHRwczovL2dpdGh1Yi5jb20vc25ha2VyczQvc2lsZXJvLXZhZC4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgICMgTW9kZWwgbG9hZGluZyBrd2FyZ3M6CiAgICAgICAgdXNlX29ubng6IGJvb2wgPSBUcnVlLAogICAgICAgIGZvcmNlX29ubnhfY3B1OiBib29sID0gVHJ1ZSwKICAgICAgICAjIERldGVjdGlvbiBrd2FyZ3M6CiAgICAgICAgdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgICAgICBzYW1wbGluZ19yYXRlOiBpbnQgPSAxNl8wMDAsCiAgICAgICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tczogaW50ID0gMjUwLAogICAgICAgIG1heF9zcGVlY2hfZHVyYXRpb25fczogZmxvYXQgPSBmbG9hdCgiaW5mIiksCiAgICAgICAgbWluX3NpbGVuY2VfZHVyYXRpb25fbXM6IGludCA9IDEwMCwKICAgICAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzOiBpbnQgPSA1MTIsCiAgICAgICAgc3BlZWNoX3BhZF9tczogaW50ID0gMzAsCiAgICAgICAgcmV0dXJuX3NlY29uZHM6IGJvb2wgPSBGYWxzZSwKICAgICAgICBwZXJfY2hhbm5lbDogYm9vbCA9IEZhbHNlLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSB2b2ljZSBhY3Rpdml0eSBkZXRlY3Rvci4KCiAgICAgICAgOnBhcmFtIHVzZV9vbm54OiAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHVzZSBPTk5YIGZvciBpbmZlcmVuY2UuIERlZmF1bHQgaXMgVHJ1ZS4KICAgICAgICA6cGFyYW0gZm9yY2Vfb25ueF9jcHU6ICAgICAgICAgIFdoZXRoZXIgdG8gZm9yY2UgT05OWCB0byB1c2UgQ1BVIGZvciBpbmZlcmVuY2UuIERlZmF1bHQgaXMgVHJ1ZS4KICAgICAgICA6cGFyYW0gdGhyZXNob2xkOiAgICAgICAgICAgICAgIFNwZWVjaCB0aHJlc2hvbGQuIFNpbGVybyBWQUQgb3V0cHV0cyBzcGVlY2ggcHJvYmFiaWxpdGllcyBmb3IgZWFjaCBhdWRpbyBjaHVuaywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHByb2JhYmlsaXRpZXMgQUJPVkUgdGhpcyB2YWx1ZSBhcmUgY29uc2lkZXJlZCBhcyBTUEVFQ0guIEl0IGlzIGJldHRlciB0byB0dW5lCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGlzIHBhcmFtZXRlciBmb3IgZWFjaCBkYXRhc2V0IHNlcGFyYXRlbHksIGJ1dCAibGF6eSIgMC41IGlzIHByZXR0eSBnb29kIGZvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9zdCBkYXRhc2V0cy4KICAgICAgICA6cGFyYW0gc2FtcGxpbmdfcmF0ZTogICAgICAgICAgIEN1cnJlbnRseSwgc2lsZXJvIFZBRCBtb2RlbHMgc3VwcG9ydCA4MDAwIGFuZCAxNjAwMCBzYW1wbGUgcmF0ZXMuCiAgICAgICAgOnBhcmFtIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM6ICBGaW5hbCBzcGVlY2ggY2h1bmtzIHNob3J0ZXIgbWluX3NwZWVjaF9kdXJhdGlvbl9tcyBhcmUgdGhyb3duIG91dC4KICAgICAgICA6cGFyYW0gbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiAgIE1heGltdW0gZHVyYXRpb24gb2Ygc3BlZWNoIGNodW5rcyBpbiBzZWNvbmRzLiBDaHVua3MgbG9uZ2VyIHRoYW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtYXhfc3BlZWNoX2R1cmF0aW9uX3NgIHdpbGwgYmUgc3BsaXQgYXQgdGhlIHRpbWVzdGFtcCBvZiB0aGUgbGFzdCBzaWxlbmNlIHRoYXQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxhc3RzIG1vcmUgdGhhbiAxMDBtcyAoaWYgYW55KSwgdG8gcHJldmVudCBhZ2dyZXNzaXZlIGN1dHRpbmcuIE90aGVyd2lzZSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZXkgd2lsbCBiZSBzcGxpdCBhZ2dyZXNzaXZlbHkganVzdCBiZWZvcmUgbWF4X3NwZWVjaF9kdXJhdGlvbl9zLgogICAgICAgIDpwYXJhbSBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogSW4gdGhlIGVuZCBvZiBlYWNoIHNwZWVjaCBjaHVuayB3YWl0IGZvciBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyBiZWZvcmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNlcGFyYXRpbmcgaXQuCiAgICAgICAgOnBhcmFtIHdpbmRvd19zaXplX3NhbXBsZXM6ICAgICBBdWRpbyBjaHVua3Mgb2Ygd2luZG93X3NpemVfc2FtcGxlcyBzaXplIGFyZSBmZWQgdG8gdGhlIHNpbGVybyBWQUQgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXQVJOSU5HISBTaWxlcm8gVkFEIG1vZGVscyB3ZXJlIHRyYWluZWQgdXNpbmcgNTEyLCAxMDI0LCAxNTM2IHNhbXBsZXMgZm9yIDE2MDAwCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgcmF0ZSBhbmQgMjU2LCA1MTIsIDc2OCBzYW1wbGVzIGZvciA4MDAwIHNhbXBsZSByYXRlLiBWYWx1ZXMgb3RoZXIgdGhhbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlc2UgbWF5IGFmZmVjdCBtb2RlbCBwZXJmb3JtYW5jZSEKICAgICAgICA6cGFyYW0gc3BlZWNoX3BhZF9tczogICAgICAgICAgIEZpbmFsIHNwZWVjaCBjaHVua3MgYXJlIHBhZGRlZCBieSBzcGVlY2hfcGFkX21zIGVhY2ggc2lkZS4KICAgICAgICA6cGFyYW0gcmV0dXJuX3NlY29uZHM6ICAgICAgICAgIFdoZXRoZXIgcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2Vjb25kcy4gRmFsc2UgbWVhbnMgdG8gcmV0dXJuIHRpbWVzdGFtcHMgaW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZXMgKGRlZmF1bHQgLSBGYWxzZSkuCiAgICAgICAgOnBhcmFtIHBlcl9jaGFubmVsOiAgICAgICAgICAgICBXaGV0aGVyIHRvIHJldHVybiB0aW1lc3RhbXBzIHBlciBjaGFubmVsIChkZWZhdWx0IC0gRmFsc2UpLiBUaGlzIHdpbGwgcnVuIFZBRAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb24gZWFjaCBjaGFubmVsIHNlcGFyYXRlbHkgYW5kIHJldHVybiBhIGxpc3Qgb2YgdGltZXN0YW1wcyBwZXIgY2hhbm5lbC4KICAgICAgICAiIiIKICAgICAgICAjIFN0b3JlIGNvbmZpZ3VyYXRpb25zOgogICAgICAgIHNlbGYuX3VzZV9vbm54ID0gdXNlX29ubngKICAgICAgICBzZWxmLl9mb3JjZV9vbm54X2NwdSA9IGZvcmNlX29ubnhfY3B1CiAgICAgICAgc2VsZi5fdGhyZXNob2xkID0gdGhyZXNob2xkCiAgICAgICAgc2VsZi5fc2FtcGxpbmdfcmF0ZSA9IHNhbXBsaW5nX3JhdGUKICAgICAgICBzZWxmLl9taW5fc3BlZWNoX2R1cmF0aW9uX21zID0gbWluX3NwZWVjaF9kdXJhdGlvbl9tcwogICAgICAgIHNlbGYuX21heF9zcGVlY2hfZHVyYXRpb25fcyA9IG1heF9zcGVlY2hfZHVyYXRpb25fcwogICAgICAgIHNlbGYuX21pbl9zaWxlbmNlX2R1cmF0aW9uX21zID0gbWluX3NpbGVuY2VfZHVyYXRpb25fbXMKICAgICAgICBzZWxmLl93aW5kb3dfc2l6ZV9zYW1wbGVzID0gd2luZG93X3NpemVfc2FtcGxlcwogICAgICAgIHNlbGYuX3NwZWVjaF9wYWRfbXMgPSBzcGVlY2hfcGFkX21zCiAgICAgICAgc2VsZi5fcmV0dXJuX3NlY29uZHMgPSByZXR1cm5fc2Vjb25kcwogICAgICAgIHNlbGYuX3Blcl9jaGFubmVsID0gcGVyX2NoYW5uZWwKCiAgICAgICAgIyBQcmVwYXJlIHRoZSBtb2RlbCB2YXJpYWJsZXMKICAgICAgICBzZWxmLl9tb2RlbDogdG9yY2guTW9kdWxlID0gTm9uZQogICAgICAgIHNlbGYuX2dldF9zcGVlY2hfdGltZXN0YW1wczogRnVuY3Rpb25UeXBlID0gTm9uZQoKICAgIGRlZiBsb2FkKHNlbGYsIGZvcmNlX3JlbG9hZDogYm9vbCA9IFRydWUpOgogICAgICAgICIiIgogICAgICAgIExvYWQgdGhlIFZBRCBtb2RlbC4KCiAgICAgICAgOnBhcmFtIGZvcmNlX3JlbG9hZDogV2hldGhlciB0byBmb3JjZSByZWxvYWQgdGhlIG1vZGVsIGV2ZW4gaWYgaXQgd2FzIGFscmVhZHkgbG9hZGVkLiBEZWZhdWx0IGlzIFRydWUuCiAgICAgICAgIiIiCiAgICAgICAgbW9kZWwsIHV0aWxzID0gdG9yY2guaHViLmxvYWQoCiAgICAgICAgICAgIHJlcG9fb3JfZGlyPSJzbmFrZXJzNC9zaWxlcm8tdmFkIiwKICAgICAgICAgICAgbW9kZWw9InNpbGVyb192YWQiLAogICAgICAgICAgICBmb3JjZV9yZWxvYWQ9Zm9yY2VfcmVsb2FkLAogICAgICAgICAgICBvbm54PXNlbGYuX3VzZV9vbm54LAogICAgICAgICAgICBmb3JjZV9vbm54X2NwdT1zZWxmLl9mb3JjZV9vbm54X2NwdSwKICAgICAgICApCiAgICAgICAgc2VsZi5fbW9kZWwgPSBtb2RlbAogICAgICAgICgKICAgICAgICAgICAgc2VsZi5fZ2V0X3NwZWVjaF90aW1lc3RhbXBzLAogICAgICAgICAgICBfLCAgIyBzYXZlX2F1ZGlvLAogICAgICAgICAgICBfLCAgIyByZWFkX2F1ZGlvLAogICAgICAgICAgICBfLCAgIyBWQURJdGVyYXRvciwKICAgICAgICAgICAgXywgICMgY29sbGVjdF9jaHVua3MKICAgICAgICApID0gdXRpbHMKCiAgICBkZWYgZGV0ZWN0X3ZvaWNlKAogICAgICAgIHNlbGYsCiAgICAgICAgYXVkaW9fZmlsZTogUGF0aCwKICAgICkgLT4gVW5pb25bTGlzdFtEaWN0W3N0ciwgaW50XV0sIExpc3RbTGlzdFtEaWN0W3N0ciwgaW50XV1dXToKICAgICAgICAiIiIKICAgICAgICBJbmZlciB0aGUgYXVkaW8gdGhyb3VnaCB0aGUgVkFEIG1vZGVsIGFuZCByZXR1cm4gdGhlIHNwZWVjaCB0aW1lc3RhbXBzLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogVGhlIGF1ZGlvIGZpbGUgdG8gaW5mZXIuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgc3BlZWNoIHRpbWVzdGFtcHMgaW4gdGhlIGF1ZGlvLiBBIGxpc3Qgb2YgdGltZXN0YW1wcyB3aGVyZSBlYWNoIHRpbWVzdGFtcCBpcyBhIGRpY3Rpb25hcnkgd2l0aCB0aGUKICAgICAgICAgICAgICAgICBmb2xsb3dpbmcga2V5czoKCiAgICAgICAgICAgICAgICAgKiAic3RhcnQiOiBUaGUgc3RhcnQgc2FtcGxlIGluZGV4IG9mIHRoZSBzcGVlY2ggaW4gdGhlIGF1ZGlvLgogICAgICAgICAgICAgICAgICogImVuZCI6ICAgVGhlIGVuZCBzYW1wbGUgaW5kZXggb2YgdGhlIHNwZWVjaCBpbiB0aGUgYXVkaW8uCgogICAgICAgICAgICAgICAgIElmIGBwZXJfY2hhbm5lbGAgaXMgVHJ1ZSwgYSBsaXN0IG9mIHRpbWVzdGFtcHMgcGVyIGNoYW5uZWwgd2lsbCBiZSByZXR1cm5lZC4KICAgICAgICAiIiIKICAgICAgICAjIENhc3QgdG8gYSBudW1weSBhcnJheToKICAgICAgICBhdWRpbyA9IHNlbGYuX3JlYWRfYXVkaW8oYXVkaW9fZmlsZSkKCiAgICAgICAgIyBEZXRlY3Qgc3BlZWNoOgogICAgICAgIGlmIG5vdCBzZWxmLl9wZXJfY2hhbm5lbDoKICAgICAgICAgICAgcmV0dXJuIHNlbGYuX2dldF9zcGVlY2hfdGltZXN0YW1wcygKICAgICAgICAgICAgICAgIGF1ZGlvLAogICAgICAgICAgICAgICAgc2VsZi5fbW9kZWwsCiAgICAgICAgICAgICAgICB0aHJlc2hvbGQ9c2VsZi5fdGhyZXNob2xkLAogICAgICAgICAgICAgICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tcz1zZWxmLl9taW5fc3BlZWNoX2R1cmF0aW9uX21zLAogICAgICAgICAgICAgICAgbWF4X3NwZWVjaF9kdXJhdGlvbl9zPXNlbGYuX21heF9zcGVlY2hfZHVyYXRpb25fcywKICAgICAgICAgICAgICAgIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zPXNlbGYuX21pbl9zaWxlbmNlX2R1cmF0aW9uX21zLAogICAgICAgICAgICAgICAgc3BlZWNoX3BhZF9tcz1zZWxmLl9zcGVlY2hfcGFkX21zLAogICAgICAgICAgICAgICAgc2FtcGxpbmdfcmF0ZT1zZWxmLl9zYW1wbGluZ19yYXRlLAogICAgICAgICAgICAgICAgd2luZG93X3NpemVfc2FtcGxlcz1zZWxmLl93aW5kb3dfc2l6ZV9zYW1wbGVzLAogICAgICAgICAgICAgICAgcmV0dXJuX3NlY29uZHM9c2VsZi5fcmV0dXJuX3NlY29uZHMsCiAgICAgICAgICAgICkKCiAgICAgICAgIyBQZXIgY2hhbm5lbDoKICAgICAgICBzcGVlY2hfdGltZXN0YW1wcyA9IFtdCiAgICAgICAgZm9yIGNoYW5uZWwgaW4gYXVkaW86CiAgICAgICAgICAgIHNwZWVjaF90aW1lc3RhbXBzLmFwcGVuZCgKICAgICAgICAgICAgICAgIHNlbGYuX2dldF9zcGVlY2hfdGltZXN0YW1wcygKICAgICAgICAgICAgICAgICAgICBjaGFubmVsLAogICAgICAgICAgICAgICAgICAgIHNlbGYuX21vZGVsLAogICAgICAgICAgICAgICAgICAgIHRocmVzaG9sZD1zZWxmLl90aHJlc2hvbGQsCiAgICAgICAgICAgICAgICAgICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tcz1zZWxmLl9taW5fc3BlZWNoX2R1cmF0aW9uX21zLAogICAgICAgICAgICAgICAgICAgIG1heF9zcGVlY2hfZHVyYXRpb25fcz1zZWxmLl9tYXhfc3BlZWNoX2R1cmF0aW9uX3MsCiAgICAgICAgICAgICAgICAgICAgbWluX3NpbGVuY2VfZHVyYXRpb25fbXM9c2VsZi5fbWluX3NpbGVuY2VfZHVyYXRpb25fbXMsCiAgICAgICAgICAgICAgICAgICAgc3BlZWNoX3BhZF9tcz1zZWxmLl9zcGVlY2hfcGFkX21zLAogICAgICAgICAgICAgICAgICAgIHNhbXBsaW5nX3JhdGU9c2VsZi5fc2FtcGxpbmdfcmF0ZSwKICAgICAgICAgICAgICAgICAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzPXNlbGYuX3dpbmRvd19zaXplX3NhbXBsZXMsCiAgICAgICAgICAgICAgICAgICAgcmV0dXJuX3NlY29uZHM9c2VsZi5fcmV0dXJuX3NlY29uZHMsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICkKCiAgICAgICAgcmV0dXJuIHNwZWVjaF90aW1lc3RhbXBzCgogICAgZGVmIF9yZWFkX2F1ZGlvKAogICAgICAgIHNlbGYsCiAgICAgICAgcGF0aDogUGF0aCwKICAgICkgLT4gdG9yY2guVGVuc29yOgogICAgICAgICIiIgogICAgICAgIFJlYWQgdGhlIGF1ZGlvIGZyb20gdGhlIGdpdmVuIHBhdGggYW5kIHJldHVybiBpdCBhcyBhIHRlbnNvci4KCiAgICAgICAgOnBhcmFtIHBhdGg6IFRoZSBwYXRoIHRvIHRoZSBhdWRpbyBmaWxlLgoKICAgICAgICA6cmV0dXJuczogVGhlIGF1ZGlvIGFzIGEgdGVuc29yLgogICAgICAgICIiIgogICAgICAgICMgUmVhZCB0aGUgYXVkaW86CiAgICAgICAgYXVkaW8sIHNhbXBsaW5nX3JhdGUgPSB0b3JjaGF1ZGlvLmxvYWQoc3RyKHBhdGgpKQoKICAgICAgICAjIENoZWNrIGlmIHRoZSBhdWRpbyBpcyBzdGVyZW8gYW5kIGlmIHNvLCBjb252ZXJ0IGl0IHRvIG1vbm8gKG9ubHkgaWYgbm90IHBlciBjaGFubmVsKToKICAgICAgICBpZiBhdWRpby5zaXplKDApID4gMSBhbmQgbm90IHNlbGYuX3Blcl9jaGFubmVsOgogICAgICAgICAgICBhdWRpbyA9IGF1ZGlvLm1lYW4oZGltPTAsIGtlZXBkaW09VHJ1ZSkKCiAgICAgICAgIyBSZXNhbXBsZSB0aGUgYXVkaW8gaWYgbmVlZGVkOgogICAgICAgIGlmIHNhbXBsaW5nX3JhdGUgIT0gc2VsZi5fc2FtcGxpbmdfcmF0ZToKICAgICAgICAgICAgdHJhbnNmb3JtID0gdG9yY2hhdWRpby50cmFuc2Zvcm1zLlJlc2FtcGxlKAogICAgICAgICAgICAgICAgb3JpZ19mcmVxPXNhbXBsaW5nX3JhdGUsIG5ld19mcmVxPXNlbGYuX3NhbXBsaW5nX3JhdGUKICAgICAgICAgICAgKQogICAgICAgICAgICBhdWRpbyA9IHRyYW5zZm9ybShhdWRpbykKCiAgICAgICAgIyBSZXR1cm4gdGhlIGF1ZGlvIChzcXVlZXplIGlmIG5vdCBwZXIgY2hhbm5lbCk6CiAgICAgICAgcmV0dXJuIGF1ZGlvIGlmIHNlbGYuX3Blcl9jaGFubmVsIGVsc2UgYXVkaW8uc3F1ZWV6ZSgwKQoKCiM6IFRoZSB2YWx1ZSB0byBzZW5kIGludG8gbXVsdGlwcm9jZXNzaW5nIHF1ZXVlcyB0byBzdG9wIHRoZSBwcm9jZXNzOgpfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSyA9ICJTVE9QIgoKCmRlZiBfbXVsdGlwcm9jZXNzaW5nX2NvbXBsZXRlX3Rhc2tzKAogICAgdmFkX2luaXRfa3dhcmdzOiBkaWN0LCB0YXNrc19xdWV1ZTogUXVldWUsIHJlc3VsdHNfcXVldWU6IFF1ZXVlCik6CiAgICAiIiIKICAgIENvbXBsZXRlIHRoZSB0YXNrcyBpbiB0aGUgZ2l2ZW4gcXVldWUgYW5kIHB1dCB0aGUgcmVzdWx0cyBpbiB0aGUgZ2l2ZW4gcmVzdWx0cyBxdWV1ZS4gVGhlIGZ1bmN0aW9uIHdpbGwgc3RvcCB3aGVuCiAgICB0aGUgZ2l2ZW4gdGFza3MgcXVldWUgd2lsbCByZWNlaXZlIHRoZSBzdG9wIG1hcmsuIEl0IGlzIGFpbWVkIHRvIGJlIHVzZWQgd2l0aCBtdWx0aXByb2Nlc3NpbmcgYXMgYSBwcm9jZXNzLgoKICAgIDpwYXJhbSB2YWRfaW5pdF9rd2FyZ3M6IFRoZSBWQUQgaW5pdGlhbGl6YXRpb24ga3dhcmdzLgogICAgOnBhcmFtIHRhc2tzX3F1ZXVlOiAgICAgQSBxdWV1ZSB0byBnZXQgdGhlIHRhc2tzIGZyb20uCiAgICA6cGFyYW0gcmVzdWx0c19xdWV1ZTogICBBIHF1ZXVlIHRvIHB1dCB0aGUgcmVzdWx0cyBpbi4KICAgICIiIgogICAgIyBJbml0aWFsaXplIGFuZCBsb2FkIHRoZSBWQUQ6CiAgICB2YWQgPSBWb2ljZUFjdGl2aXR5RGV0ZWN0b3IoKip2YWRfaW5pdF9rd2FyZ3MpCiAgICB2YWQubG9hZChmb3JjZV9yZWxvYWQ9RmFsc2UpCgogICAgIyBTdGFydCBsaXN0ZW5pbmcgdG8gdGhlIHRhc2tzIHF1ZXVlOgogICAgd2hpbGUgVHJ1ZToKICAgICAgICAjIEdldCB0aGUgdGFzazoKICAgICAgICB0YXNrOiBUdXBsZVtzdHIsIGRpY3RdID0gdGFza3NfcXVldWUuZ2V0KCkKICAgICAgICBpZiB0YXNrID09IF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLOgogICAgICAgICAgICBicmVhawogICAgICAgIHRyeToKICAgICAgICAgICAgIyBDcmVhdGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2sgPSBUYXNrQ3JlYXRvci5mcm9tX3R1cGxlKHRhc2tfdHVwbGU9dGFzaykKICAgICAgICAgICAgIyBSdW4gdGhlIGZpbGUgdGhyb3VnaCB0aGUgVkFEOgogICAgICAgICAgICBzcGVlY2hfdGltZXN0YW1wcyA9IHZhZC5kZXRlY3Rfdm9pY2UoYXVkaW9fZmlsZT10YXNrLmF1ZGlvX2ZpbGUpCiAgICAgICAgICAgICMgQ29tcGxldGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2suZG9fdGFzayhzcGVlY2hfdGltZXN0YW1wcz1zcGVlY2hfdGltZXN0YW1wcykKICAgICAgICAgICAgIyBCdWlsZCB0aGUgcmVzdWx0OgogICAgICAgICAgICByZXN1bHQgPSAoRmFsc2UsIHRhc2suZ2V0X3Jlc3VsdCgpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIEJ1aWxkIHRoZSBlcnJvcjoKICAgICAgICAgICAgcmVzdWx0ID0gKFRydWUsICh0YXNrLmF1ZGlvX2ZpbGUubmFtZSwgc3RyKGV4Y2VwdGlvbikpKQogICAgICAgICMgQ29sbGVjdCB0aGUgcmVzdWx0IC8gZXJyb3I6CiAgICAgICAgcmVzdWx0c19xdWV1ZS5wdXQocmVzdWx0KQoKICAgICMgTWFyayB0aGUgZW5kIG9mIHRoZSB0YXNrczoKICAgIHJlc3VsdHNfcXVldWUucHV0KF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLKQoKCiMgR2V0IHRoZSBnbG9iYWwgbG9nZ2VyOgp0cnk6CiAgICBpbXBvcnQgbWxydW4KCiAgICBfTE9HR0VSID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9jdHgoInNpbGVyb192YWQiKS5sb2dnZXIKZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3I6CiAgICBfTE9HR0VSID0gbG9nZ2luZy5nZXRMb2dnZXIoKQoKCmRlZiBkZXRlY3Rfdm9pY2UoCiAgICAjIElucHV0IGt3YXJnczoKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBQYXRoLCBMaXN0W1VuaW9uW3N0ciwgUGF0aF1dXSwKICAgICMgTW9kZWwgbG9hZGluZyBrd2FyZ3M6CiAgICB1c2Vfb25ueDogYm9vbCA9IFRydWUsCiAgICBmb3JjZV9vbm54X2NwdTogYm9vbCA9IFRydWUsCiAgICAjIERldGVjdGlvbiBrd2FyZ3M6CiAgICB0aHJlc2hvbGQ6IGZsb2F0ID0gMC41LAogICAgc2FtcGxpbmdfcmF0ZTogaW50ID0gMTZfMDAwLAogICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tczogaW50ID0gMjUwLAogICAgbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiBmbG9hdCA9IGZsb2F0KCJpbmYiKSwKICAgIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBpbnQgPSAxMDAsCiAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzOiBpbnQgPSA1MTIsCiAgICBzcGVlY2hfcGFkX21zOiBpbnQgPSAzMCwKICAgIHJldHVybl9zZWNvbmRzOiBib29sID0gRmFsc2UsCiAgICBwZXJfY2hhbm5lbDogYm9vbCA9IEZhbHNlLAogICAgIyBPdGhlciBrd2FyZ3M6CiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBQZXJmb3JtIHZvaWNlIGFjdGl2aXR5IGRldGVjdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyB0aGUgc2lsZXJvIFZBRCBtb2RlbCAtCiAgICBodHRwczovL2dpdGh1Yi5jb20vc25ha2VyczQvc2lsZXJvLXZhZC4gVGhlIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIKICAgIFZBRCB0aW1lc3RhbXBzIGRpY3Rpb25hcmllcyBhcyB2YWx1ZS4KCiAgICBGb3IgZXhhbXBsZTo6CgogICAgICAgIHsKICAgICAgICAgICAgImZpbGVfMS53YXYiOiBbCiAgICAgICAgICAgICAgICB7InN0YXJ0IjogMCwgImVuZCI6IDE2MDAwfSwKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAxNjAwMCwgImVuZCI6IDMyMDAwfSwKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAzMjAwMCwgImVuZCI6IDQ4MDAwfSwKICAgICAgICAgICAgICAgIC4uLgogICAgICAgICAgICBdLAogICAgICAgICAgICAiZmlsZV8yLndhdiI6IFsKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAwLCAiZW5kIjogMTYwMDB9LAogICAgICAgICAgICAgICAgeyJzdGFydCI6IDE2MDAwLCAiZW5kIjogMzIwMDB9LAogICAgICAgICAgICAgICAgeyJzdGFydCI6IDMyMDAwLCAiZW5kIjogNDgwMDB9LAogICAgICAgICAgICAgICAgLi4uCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgIC4uLgogICAgICAgIH0KCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgICAgICBUaGUgcGF0aCB0byB0aGUgYXVkaW8gZmlsZXMgdG8gZGlhcml6ZS4gQ2FuIGJlIGEgcGF0aCB0byBhIHNpbmdsZSBmaWxlLCBhIHBhdGggdG8gYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkaXJlY3Rvcnkgb3IgYSBsaXN0IG9mIHBhdGhzIHRvIGZpbGVzLgogICAgOnBhcmFtIHVzZV9vbm54OiAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHVzZSBPTk5YIGZvciBpbmZlcmVuY2UuIERlZmF1bHQgaXMgVHJ1ZS4KICAgIDpwYXJhbSBmb3JjZV9vbm54X2NwdTogICAgICAgICAgV2hldGhlciB0byBmb3JjZSBPTk5YIHRvIHVzZSBDUFUgZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgOnBhcmFtIHRocmVzaG9sZDogICAgICAgICAgICAgICBTcGVlY2ggdGhyZXNob2xkLiBTaWxlcm8gVkFEIG91dHB1dHMgc3BlZWNoIHByb2JhYmlsaXRpZXMgZm9yIGVhY2ggYXVkaW8gY2h1bmssCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHByb2JhYmlsaXRpZXMgQUJPVkUgdGhpcyB2YWx1ZSBhcmUgY29uc2lkZXJlZCBhcyBTUEVFQ0guIEl0IGlzIGJldHRlciB0byB0dW5lCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoaXMgcGFyYW1ldGVyIGZvciBlYWNoIGRhdGFzZXQgc2VwYXJhdGVseSwgYnV0ICJsYXp5IiAwLjUgaXMgcHJldHR5IGdvb2QgZm9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vc3QgZGF0YXNldHMuCiAgICA6cGFyYW0gc2FtcGxpbmdfcmF0ZTogICAgICAgICAgIEN1cnJlbnRseSwgc2lsZXJvIFZBRCBtb2RlbHMgc3VwcG9ydCA4MDAwIGFuZCAxNjAwMCBzYW1wbGUgcmF0ZXMuCiAgICA6cGFyYW0gbWluX3NwZWVjaF9kdXJhdGlvbl9tczogIEZpbmFsIHNwZWVjaCBjaHVua3Mgc2hvcnRlciBtaW5fc3BlZWNoX2R1cmF0aW9uX21zIGFyZSB0aHJvd24gb3V0LgogICAgOnBhcmFtIG1heF9zcGVlY2hfZHVyYXRpb25fczogICBNYXhpbXVtIGR1cmF0aW9uIG9mIHNwZWVjaCBjaHVua3MgaW4gc2Vjb25kcy4gQ2h1bmtzIGxvbmdlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtYXhfc3BlZWNoX2R1cmF0aW9uX3NgIHdpbGwgYmUgc3BsaXQgYXQgdGhlIHRpbWVzdGFtcCBvZiB0aGUgbGFzdCBzaWxlbmNlIHRoYXQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGFzdHMgbW9yZSB0aGFuIDEwMG1zIChpZiBhbnkpLCB0byBwcmV2ZW50IGFnZ3Jlc3NpdmUgY3V0dGluZy4gT3RoZXJ3aXNlLCB0aGV5IHdpbGwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYmUgc3BsaXQgYWdncmVzc2l2ZWx5IGp1c3QgYmVmb3JlIG1heF9zcGVlY2hfZHVyYXRpb25fcy4KICAgIDpwYXJhbSBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogSW4gdGhlIGVuZCBvZiBlYWNoIHNwZWVjaCBjaHVuayB3YWl0IGZvciBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyBiZWZvcmUgc2VwYXJhdGluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpdC4KICAgIDpwYXJhbSB3aW5kb3dfc2l6ZV9zYW1wbGVzOiAgICAgQXVkaW8gY2h1bmtzIG9mIHdpbmRvd19zaXplX3NhbXBsZXMgc2l6ZSBhcmUgZmVkIHRvIHRoZSBzaWxlcm8gVkFEIG1vZGVsLgoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV0FSTklORyEgU2lsZXJvIFZBRCBtb2RlbHMgd2VyZSB0cmFpbmVkIHVzaW5nIDUxMiwgMTAyNCwgMTUzNiBzYW1wbGVzIGZvciAxNjAwMAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgcmF0ZSBhbmQgMjU2LCA1MTIsIDc2OCBzYW1wbGVzIGZvciA4MDAwIHNhbXBsZSByYXRlLiBWYWx1ZXMgb3RoZXIgdGhhbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGVzZSBtYXkgYWZmZWN0IG1vZGVsIHBlcmZvcm1hbmNlIQogICAgOnBhcmFtIHNwZWVjaF9wYWRfbXM6ICAgICAgICAgICBGaW5hbCBzcGVlY2ggY2h1bmtzIGFyZSBwYWRkZWQgYnkgc3BlZWNoX3BhZF9tcyBlYWNoIHNpZGUuCiAgICA6cGFyYW0gcmV0dXJuX3NlY29uZHM6ICAgICAgICAgIFdoZXRoZXIgcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2Vjb25kcy4gRmFsc2UgbWVhbnMgdG8gcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2FtcGxlcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoZGVmYXVsdCAtIEZhbHNlKS4KICAgIDpwYXJhbSBwZXJfY2hhbm5lbDogICAgICAgICAgICAgV2hldGhlciB0byByZXR1cm4gdGltZXN0YW1wcyBwZXIgY2hhbm5lbCAoZGVmYXVsdCAtIEZhbHNlKS4gVGhpcyB3aWxsIHJ1biBWQUQgb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZWFjaCBjaGFubmVsIHNlcGFyYXRlbHkgYW5kIHJldHVybiBhIGxpc3Qgb2YgdGltZXN0YW1wcyBwZXIgY2hhbm5lbC4KICAgIDpwYXJhbSB1c2VfbXVsdGlwcm9jZXNzaW5nOiAgICAgVGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHVzZSBmb3IgbXVsdGlwcm9jZXNzaW5nLiBJZiAwLCBubyBtdWx0aXByb2Nlc3Npbmcgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkLiBEZWZhdWx0IGlzIDAuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgICAgIFZlcmJvc2l0eS4KICAgICIiIgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICAjIEdldCB0aGUgaW5wdXQgYXVkaW8gZmlsZXMgdG8gdHJhbnNjcmliZToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJDb2xsZWN0aW5nIGF1ZGlvIGZpbGVzLiIpCiAgICBhdWRpb19maWxlcyA9IF9nZXRfYXVkaW9fZmlsZXMoZGF0YV9wYXRoPWRhdGFfcGF0aCkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiQ29sbGVjdGVkIHtsZW4oYXVkaW9fZmlsZXMpfSBhdWRpbyBmaWxlcy4iKQoKICAgICMgSW5pdGlhbGl6ZSB0aGUgdHJhbnNjcmlwdGlvbiBwaXBlbGluZToKICAgIHZhZF9pbml0X2t3YXJncyA9IHsKICAgICAgICAidXNlX29ubngiOiB1c2Vfb25ueCwKICAgICAgICAiZm9yY2Vfb25ueF9jcHUiOiBmb3JjZV9vbm54X2NwdSwKICAgICAgICAidGhyZXNob2xkIjogdGhyZXNob2xkLAogICAgICAgICJzYW1wbGluZ19yYXRlIjogc2FtcGxpbmdfcmF0ZSwKICAgICAgICAibWluX3NwZWVjaF9kdXJhdGlvbl9tcyI6IG1pbl9zcGVlY2hfZHVyYXRpb25fbXMsCiAgICAgICAgIm1heF9zcGVlY2hfZHVyYXRpb25fcyI6IG1heF9zcGVlY2hfZHVyYXRpb25fcywKICAgICAgICAibWluX3NpbGVuY2VfZHVyYXRpb25fbXMiOiBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcywKICAgICAgICAid2luZG93X3NpemVfc2FtcGxlcyI6IHdpbmRvd19zaXplX3NhbXBsZXMsCiAgICAgICAgInNwZWVjaF9wYWRfbXMiOiBzcGVlY2hfcGFkX21zLAogICAgICAgICJyZXR1cm5fc2Vjb25kcyI6IHJldHVybl9zZWNvbmRzLAogICAgICAgICJwZXJfY2hhbm5lbCI6IHBlcl9jaGFubmVsLAogICAgfQoKICAgICMgQ3JlYXRlIHRoZSB0YXNrIGNyZWF0b3I6CiAgICB0YXNrX2NyZWF0b3IgPSBUYXNrQ3JlYXRvcih0YXNrX3R5cGU9QmFzZVRhc2spCgogICAgIyBSdW4gdGhlIHRyYW5zY3JpcHRpb246CiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEZXRlY3Rpbmcgdm9pY2UiLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEZXRlY3Rpbmcgdm9pY2UiLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgICMgUHJvY2VzcyB0aGUgcmVzdWx0czoKICAgIHJldHVybiBfcHJvY2Vzc19yZXN1bHRzKHJlc3VsdHM9cmVzdWx0cywgdmVyYm9zZT12ZXJib3NlKQoKCmRlZiBkaWFyaXplKAogICAgIyBJbnB1dCAvIE91dHB1dCBrd2FyZ3M6CiAgICBkYXRhX3BhdGg6IFVuaW9uW3N0ciwgUGF0aCwgTGlzdFtVbmlvbltzdHIsIFBhdGhdXV0sCiAgICAjIE1vZGVsIGxvYWRpbmcga3dhcmdzOgogICAgdXNlX29ubng6IGJvb2wgPSBUcnVlLAogICAgZm9yY2Vfb25ueF9jcHU6IGJvb2wgPSBUcnVlLAogICAgIyBEZXRlY3Rpb24ga3dhcmdzOgogICAgdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgIHNhbXBsaW5nX3JhdGU6IGludCA9IDE2XzAwMCwKICAgIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM6IGludCA9IDI1MCwKICAgIG1heF9zcGVlY2hfZHVyYXRpb25fczogZmxvYXQgPSBmbG9hdCgiaW5mIiksCiAgICBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogaW50ID0gMTAwLAogICAgd2luZG93X3NpemVfc2FtcGxlczogaW50ID0gNTEyLAogICAgc3BlZWNoX3BhZF9tczogaW50ID0gMzAsCiAgICAjIERpYXJpemF0aW9uIGt3YXJnczoKICAgIHNwZWFrZXJfbGFiZWxzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgIyBPdGhlciBrd2FyZ3M6CiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBQZXJmb3JtIHNwZWVjaCBkaWFyaXphdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyB0aGUgc2lsZXJvIFZBRCBtb2RlbCAtIGh0dHBzOi8vZ2l0aHViLmNvbS9zbmFrZXJzNC9zaWxlcm8tdmFkLgogICAgVGhlIHNwZWVjaCBkaWFyaXphdGlvbiBpcyBwZXJmb3JtZWQgcGVyIGNoYW5uZWwgc28gdGhhdCBlYWNoIGNoYW5uZWwgaW4gdGhlIGF1ZGlvIGJlbG9uZyB0byBhIGRpZmZlcmVudCBzcGVha2VyLiBUaGUKICAgIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIgZGlhcml6YXRpb24gYXMgdmFsdWUuIEEgZGlhcml6YXRpb24gaXMgYSBsaXN0CiAgICBvZiB0dXBsZXM6IChzdGFydCwgZW5kLCBzcGVha2VyX2xhYmVsKS4KCiAgICBGb3IgZXhhbXBsZTo6CgogICAgICAgIHsKICAgICAgICAgICAgImZpbGVfMS53YXYiOiBbCiAgICAgICAgICAgICAgICAoMC4wLCAxLjAsICJzcGVha2VyXzAiKSwKICAgICAgICAgICAgICAgICgxLjAsIDIuMCwgInNwZWFrZXJfMSIpLAogICAgICAgICAgICAgICAgKDIuMCwgMy4wLCAic3BlYWtlcl8wIiksCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgImZpbGVfMi53YXYiOiBbCiAgICAgICAgICAgICAgICAoMC4wLCAxLjAsICJzcGVha2VyXzAiKSwKICAgICAgICAgICAgICAgICgxLjAsIDIuMCwgInNwZWFrZXJfMSIpLAogICAgICAgICAgICAgICAgKDIuMCwgMy4wLCAic3BlYWtlcl8wIiksCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgLi4uCiAgICAgICAgfQoKCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICAgICAgIFRoZSBwYXRoIHRvIHRoZSBhdWRpbyBmaWxlcyB0byBkaWFyaXplLiBDYW4gYmUgYSBwYXRoIHRvIGEgc2luZ2xlIGZpbGUsIGEgcGF0aCB0byBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRpcmVjdG9yeSBvciBhIGxpc3Qgb2YgcGF0aHMgdG8gZmlsZXMuCiAgICA6cGFyYW0gdXNlX29ubng6ICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gdXNlIE9OTlggZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgOnBhcmFtIGZvcmNlX29ubnhfY3B1OiAgICAgICAgICBXaGV0aGVyIHRvIGZvcmNlIE9OTlggdG8gdXNlIENQVSBmb3IgaW5mZXJlbmNlLiBEZWZhdWx0IGlzIFRydWUuCiAgICA6cGFyYW0gdGhyZXNob2xkOiAgICAgICAgICAgICAgIFNwZWVjaCB0aHJlc2hvbGQuIFNpbGVybyBWQUQgb3V0cHV0cyBzcGVlY2ggcHJvYmFiaWxpdGllcyBmb3IgZWFjaCBhdWRpbyBjaHVuaywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcHJvYmFiaWxpdGllcyBBQk9WRSB0aGlzIHZhbHVlIGFyZSBjb25zaWRlcmVkIGFzIFNQRUVDSC4gSXQgaXMgYmV0dGVyIHRvIHR1bmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhpcyBwYXJhbWV0ZXIgZm9yIGVhY2ggZGF0YXNldCBzZXBhcmF0ZWx5LCBidXQgImxhenkiIDAuNSBpcyBwcmV0dHkgZ29vZCBmb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9zdCBkYXRhc2V0cy4KICAgIDpwYXJhbSBzYW1wbGluZ19yYXRlOiAgICAgICAgICAgQ3VycmVudGx5LCBzaWxlcm8gVkFEIG1vZGVscyBzdXBwb3J0IDgwMDAgYW5kIDE2MDAwIHNhbXBsZSByYXRlcy4KICAgIDpwYXJhbSBtaW5fc3BlZWNoX2R1cmF0aW9uX21zOiAgRmluYWwgc3BlZWNoIGNodW5rcyBzaG9ydGVyIG1pbl9zcGVlY2hfZHVyYXRpb25fbXMgYXJlIHRocm93biBvdXQuCiAgICA6cGFyYW0gbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiAgIE1heGltdW0gZHVyYXRpb24gb2Ygc3BlZWNoIGNodW5rcyBpbiBzZWNvbmRzLiBDaHVua3MgbG9uZ2VyIHRoYW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYG1heF9zcGVlY2hfZHVyYXRpb25fc2Agd2lsbCBiZSBzcGxpdCBhdCB0aGUgdGltZXN0YW1wIG9mIHRoZSBsYXN0IHNpbGVuY2UgdGhhdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBsYXN0cyBtb3JlIHRoYW4gMTAwbXMgKGlmIGFueSksIHRvIHByZXZlbnQgYWdncmVzc2l2ZSBjdXR0aW5nLiBPdGhlcndpc2UsIHRoZXkgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSBzcGxpdCBhZ2dyZXNzaXZlbHkganVzdCBiZWZvcmUgbWF4X3NwZWVjaF9kdXJhdGlvbl9zLgogICAgOnBhcmFtIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBJbiB0aGUgZW5kIG9mIGVhY2ggc3BlZWNoIGNodW5rIHdhaXQgZm9yIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zIGJlZm9yZSBzZXBhcmF0aW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGl0LgogICAgOnBhcmFtIHdpbmRvd19zaXplX3NhbXBsZXM6ICAgICBBdWRpbyBjaHVua3Mgb2Ygd2luZG93X3NpemVfc2FtcGxlcyBzaXplIGFyZSBmZWQgdG8gdGhlIHNpbGVybyBWQUQgbW9kZWwuCgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXQVJOSU5HISBTaWxlcm8gVkFEIG1vZGVscyB3ZXJlIHRyYWluZWQgdXNpbmcgNTEyLCAxMDI0LCAxNTM2IHNhbXBsZXMgZm9yIDE2MDAwCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSByYXRlIGFuZCAyNTYsIDUxMiwgNzY4IHNhbXBsZXMgZm9yIDgwMDAgc2FtcGxlIHJhdGUuIFZhbHVlcyBvdGhlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZXNlIG1heSBhZmZlY3QgbW9kZWwgcGVyZm9ybWFuY2UhCiAgICA6cGFyYW0gc3BlZWNoX3BhZF9tczogICAgICAgICAgIEZpbmFsIHNwZWVjaCBjaHVua3MgYXJlIHBhZGRlZCBieSBzcGVlY2hfcGFkX21zIGVhY2ggc2lkZS4KICAgIDpwYXJhbSBzcGVha2VyX2xhYmVsczogICAgICAgICAgVGhlIHNwZWFrZXIgbGFiZWxzIHRvIHVzZSBmb3IgdGhlIGRpYXJpemF0aW9uLiBJZiBub3QgZ2l2ZW4sIHRoZSBzcGVha2VycyB3aWxsIGJlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5hbWVkICJzcGVha2VyXzAiLCAic3BlYWtlcl8xIiwgZXRjLgogICAgOnBhcmFtIHVzZV9tdWx0aXByb2Nlc3Npbmc6ICAgICBUaGUgbnVtYmVyIG9mIHdvcmtlcnMgdG8gdXNlIGZvciBtdWx0aXByb2Nlc3NpbmcuIElmIDAsIG5vIG11bHRpcHJvY2Vzc2luZyB3aWxsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJlIHVzZWQuIERlZmF1bHQgaXMgMC4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgICAgVmVyYm9zaXR5LgogICAgIiIiCiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgR2V0IHRoZSBpbnB1dCBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgYXVkaW8gZmlsZXMuIikKICAgIGF1ZGlvX2ZpbGVzID0gX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbihhdWRpb19maWxlcyl9IGF1ZGlvIGZpbGVzLiIpCgogICAgIyBJbml0aWFsaXplIHRoZSB0cmFuc2NyaXB0aW9uIHBpcGVsaW5lOgogICAgdmFkX2luaXRfa3dhcmdzID0gewogICAgICAgICJ1c2Vfb25ueCI6IHVzZV9vbm54LAogICAgICAgICJmb3JjZV9vbm54X2NwdSI6IGZvcmNlX29ubnhfY3B1LAogICAgICAgICJ0aHJlc2hvbGQiOiB0aHJlc2hvbGQsCiAgICAgICAgInNhbXBsaW5nX3JhdGUiOiBzYW1wbGluZ19yYXRlLAogICAgICAgICJtaW5fc3BlZWNoX2R1cmF0aW9uX21zIjogbWluX3NwZWVjaF9kdXJhdGlvbl9tcywKICAgICAgICAibWF4X3NwZWVjaF9kdXJhdGlvbl9zIjogbWF4X3NwZWVjaF9kdXJhdGlvbl9zLAogICAgICAgICJtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyI6IG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zLAogICAgICAgICJ3aW5kb3dfc2l6ZV9zYW1wbGVzIjogd2luZG93X3NpemVfc2FtcGxlcywKICAgICAgICAic3BlZWNoX3BhZF9tcyI6IHNwZWVjaF9wYWRfbXMsCiAgICAgICAgInJldHVybl9zZWNvbmRzIjogVHJ1ZSwKICAgICAgICAicGVyX2NoYW5uZWwiOiBUcnVlLAogICAgfQoKICAgICMgQ3JlYXRlIHRoZSB0YXNrIGNyZWF0b3I6CiAgICB0YXNrX2NyZWF0b3IgPSBUYXNrQ3JlYXRvcigKICAgICAgICB0YXNrX3R5cGU9U3BlZWNoRGlhcml6YXRpb25UYXNrLCB0YXNrX2t3YXJncz17InNwZWFrZXJfbGFiZWxzIjogc3BlYWtlcl9sYWJlbHN9CiAgICApCgogICAgIyBSdW4gdGhlIHRyYW5zY3JpcHRpb246CiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEaWFyaXppbmciLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEaWFyaXppbmciLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgICMgUHJvY2VzcyB0aGUgcmVzdWx0czoKICAgIHJldHVybiBfcHJvY2Vzc19yZXN1bHRzKHJlc3VsdHM9cmVzdWx0cywgdmVyYm9zZT12ZXJib3NlKQoKCmRlZiBfZ2V0X2F1ZGlvX2ZpbGVzKAogICAgZGF0YV9wYXRoOiBVbmlvbltQYXRoLCBzdHIsIGxpc3RdLAopIC0+IExpc3RbUGF0aF06CiAgICAiIiIKICAgIEdldCB0aGUgYXVkaW8gZmlsZXMgZnJvbSB0aGUgZGF0YSBwYXRoLiBJZiBhIHBhdGggdG8gYSBkaXJlY3RvcnkgaXMgZ2l2ZW4sIGFsbCBmaWxlcyBpbiB0aGUgZGlyZWN0b3J5IHdpbGwgYmUKICAgIGNvbGxlY3RlZC4KCiAgICA6cGFyYW0gZGF0YV9wYXRoOiBUaGUgZGF0YSBwYXRoIHRvIGNvbGxlY3QgdGhlIGF1ZGlvIGZpbGVzIGZyb20uCgogICAgOnJldHVybnM6IFRoZSBhdWRpbyBmaWxlcyBsaXN0LgogICAgIiIiCiAgICAjIENoZWNrIGlmIGdpdmVuIGEgbGlzdCBvZiBwYXRoczoKICAgIGlmIGlzaW5zdGFuY2UoZGF0YV9wYXRoLCBsaXN0KToKICAgICAgICBhdWRpb19maWxlcyA9IFtdCiAgICAgICAgZm9yIHBhdGggaW4gZGF0YV9wYXRoOgogICAgICAgICAgICBhdWRpb19maWxlcy5leHRlbmQoX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9cGF0aCkpCiAgICAgICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgogICAgIyBDaGVjayBpZiBnaXZlbiBhIHNpbmdsZSBzdHJpbmcgcGF0aCB0byBjYXN0IGl0IHRvIGEgYHBhdGhsaWIuUGF0aGA6CiAgICBpZiBpc2luc3RhbmNlKGRhdGFfcGF0aCwgc3RyKToKICAgICAgICBkYXRhX3BhdGggPSBQYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQoKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICBhdWRpb19maWxlcyA9IGxpc3QoZGF0YV9wYXRoLmdsb2IoIiouKiIpKQogICAgZWxpZiBkYXRhX3BhdGguaXNfZmlsZSgpOgogICAgICAgIGF1ZGlvX2ZpbGVzID0gW2RhdGFfcGF0aF0KICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJVbnJlY29nbml6ZWQgZGF0YSBwYXRoLiBUaGUgcGFyYW1ldGVyIGBkYXRhX3BhdGhgIG11c3QgYmUgYSB2YWxpZCBwYXRoIHRvIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgIgogICAgICAgICAgICBmImZpbGUuIEdpdmVuOiB7c3RyKGRhdGFfcGF0aCl9ICIKICAgICAgICApCgogICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgoKZGVmIF9ydW4oCiAgICBhdWRpb19maWxlczogTGlzdFtQYXRoXSwKICAgIGRlc2NyaXB0aW9uOiBzdHIsCiAgICB2YWRfaW5pdF9rd2FyZ3M6IGRpY3QsCiAgICB0YXNrX2NyZWF0b3I6IFRhc2tDcmVhdG9yLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgbGlzdF1dXToKICAgICIiIgogICAgTG9hZCBhIFZBRCBhbmQgdXNlIGl0IHRvIGNvbXBsZXRlIHRoZSB0YXNrcyB0aGF0IHdpbGwgYmUgY3JlYXRlZCBvbiB0aGUgcHJvdmlkZWQgZmlsZXMgdXNpbmcgdGhlIGdpdmVuIHRhc2sgY3JlYXRvci4KCiAgICA6cGFyYW0gYXVkaW9fZmlsZXM6ICAgICBUaGUgYXVkaW8gZmlsZXMgdG8gdXNlLgogICAgOnBhcmFtIGRlc2NyaXB0aW9uOiAgICAgVGhlIGRlc2NyaXB0aW9uIHRvIHVzZSBmb3IgdGhlIHByb2dyZXNzIGJhci4KICAgIDpwYXJhbSB2YWRfaW5pdF9rd2FyZ3M6IFRoZSBWQUQgaW5pdGlhbGl6YXRpb24ga2V5d29yZCBhcmd1bWVudHMuCiAgICA6cGFyYW0gdGFza19jcmVhdG9yOiAgICBUaGUgdGFzayBjcmVhdG9yIHRvIHVzZSB0byBjcmVhdGUgdGhlIHRhc2tzLgogICAgOnBhcmFtIHZlcmJvc2U6ICAgICAgICAgVmVyYm9zaXR5LgoKICAgIDpyZXR1cm5zOiBUaGUgY29sbGVjdGVkIHJlc3VsdHMuCiAgICAiIiIKICAgICMgTG9hZCB0aGUgVkFEOgogICAgdmFkID0gVm9pY2VBY3Rpdml0eURldGVjdG9yKCoqdmFkX2luaXRfa3dhcmdzKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJMb2FkaW5nIHRoZSBWQUQgbW9kZWwuIikKICAgIHZhZC5sb2FkKCkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJWQUQgbW9kZWwgbG9hZGVkLiIpCgogICAgIyBSdW4gdGhlIFZBRCBvbiB0aGUgYXVkaW8gZmlsZXMgYW5kIGNvbGxlY3QgdGhlIHJlc3VsdHM6CiAgICByZXN1bHRzID0gW10KICAgIGZvciBhdWRpb19maWxlIGluIHRxZG0oCiAgICAgICAgYXVkaW9fZmlsZXMsCiAgICAgICAgZGVzYz1kZXNjcmlwdGlvbiwKICAgICAgICB1bml0PSJmaWxlIiwKICAgICAgICB0b3RhbD1sZW4oYXVkaW9fZmlsZXMpLAogICAgICAgIGRpc2FibGU9bm90IHZlcmJvc2UsCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgIyBDcmVhdGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2sgPSB0YXNrX2NyZWF0b3IuY3JlYXRlX3Rhc2soYXVkaW9fZmlsZT1hdWRpb19maWxlKQogICAgICAgICAgICAjIFJ1biB0aGUgZmlsZSB0aHJvdWdoIHRoZSBWQUQ6CiAgICAgICAgICAgIHNwZWVjaF90aW1lc3RhbXBzID0gdmFkLmRldGVjdF92b2ljZShhdWRpb19maWxlPWF1ZGlvX2ZpbGUpCiAgICAgICAgICAgICMgQ29tcGxldGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2suZG9fdGFzayhzcGVlY2hfdGltZXN0YW1wcz1zcGVlY2hfdGltZXN0YW1wcykKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSByZXN1bHQ6CiAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKChGYWxzZSwgdGFzay5nZXRfcmVzdWx0KCkpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIENvbGxlY3QgdGhlIGVycm9yOgogICAgICAgICAgICByZXN1bHRzLmFwcGVuZCgoVHJ1ZSwgKGF1ZGlvX2ZpbGUubmFtZSwgc3RyKGV4Y2VwdGlvbikpKSkKCiAgICByZXR1cm4gcmVzdWx0cwoKCmRlZiBfcGFyYWxsZWxfcnVuKAogICAgbl93b3JrZXJzOiBpbnQsCiAgICBhdWRpb19maWxlczogTGlzdFtQYXRoXSwKICAgIGRlc2NyaXB0aW9uOiBzdHIsCiAgICB2YWRfaW5pdF9rd2FyZ3M6IGRpY3QsCiAgICB0YXNrX2NyZWF0b3I6IFRhc2tDcmVhdG9yLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgbGlzdF1dXToKICAgICIiIgogICAgUnVuIG11bHRpcGxlIFZBRCB3b3JrZXJzIHdpdGggbXVsdGlwcm9jZXNzaW5nIHRvIGNvbXBsZXRlIHRoZSB0YXNrcyB0aGF0IHdpbGwgYmUgY3JlYXRlZCBvbiB0aGUgcHJvdmlkZWQgZmlsZXMgdXNpbmcKICAgIHRoZSBnaXZlbiB0YXNrIGNyZWF0b3IuCgogICAgOnBhcmFtIG5fd29ya2VyczogICAgICAgVGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHVzZS4KICAgIDpwYXJhbSBhdWRpb19maWxlczogICAgIFRoZSBhdWRpbyBmaWxlcyB0byB1c2UuCiAgICA6cGFyYW0gZGVzY3JpcHRpb246ICAgICBUaGUgZGVzY3JpcHRpb24gdG8gdXNlIGZvciB0aGUgcHJvZ3Jlc3MgYmFyLgogICAgOnBhcmFtIHZhZF9pbml0X2t3YXJnczogVGhlIFZBRCBpbml0aWFsaXphdGlvbiBrZXl3b3JkIGFyZ3VtZW50cy4KICAgIDpwYXJhbSB0YXNrX2NyZWF0b3I6ICAgIFRoZSB0YXNrIGNyZWF0b3IgdG8gdXNlIHRvIGNyZWF0ZSB0aGUgdGFza3MuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICBWZXJib3NpdHkuCgogICAgOnJldHVybnM6IFRoZSBjb2xsZWN0ZWQgcmVzdWx0cy4KICAgICIiIgogICAgIyBMb2FkIHRoZSBWQUQgKGRvd25sb2FkIG9uY2UsIGFuZCBpdCB3aWxsIGJlIGxvYWRlZCB0aGVuIHBlciBwcm9jZXNzIGxhdGVyIG9uKToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiTG9hZGluZyB0aGUgVkFEIG1vZGVsLiIpCiAgICB2YWQgPSBWb2ljZUFjdGl2aXR5RGV0ZWN0b3IoKip2YWRfaW5pdF9rd2FyZ3MpCiAgICB2YWQubG9hZCgpCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiVkFEIG1vZGVsIGxvYWRlZC4iKQoKICAgICMgQ2hlY2sgdGhlIG51bWJlciBvZiB3b3JrZXJzOgogICAgaWYgbl93b3JrZXJzID4gbGVuKGF1ZGlvX2ZpbGVzKToKICAgICAgICBfTE9HR0VSLndhcm5pbmcoCiAgICAgICAgICAgIGYiVGhlIG51bWJlciBvZiB3b3JrZXJzICh7bl93b3JrZXJzfSkgaXMgbGFyZ2VyIHRoYW4gdGhlIG51bWJlciBvZiBhdWRpbyBmaWxlcyAoe2xlbihhdWRpb19maWxlcyl9KS4gIgogICAgICAgICAgICBmIlNldHRpbmcgdGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHtsZW4oYXVkaW9fZmlsZXMpfS4iCiAgICAgICAgKQogICAgICAgIG5fd29ya2VycyA9IGxlbihhdWRpb19maWxlcykKCiAgICAjIEluaXRpYWxpemUgdGhlIG11bHRpcHJvY2Vzc2luZyBxdWV1ZXM6CiAgICB0YXNrc19xdWV1ZSA9IFF1ZXVlKCkKICAgIHJlc3VsdHNfcXVldWUgPSBRdWV1ZSgpCgogICAgIyBJbml0aWFsaXplIHRoZSBtdWx0aXByb2Nlc3NpbmcgcHJvY2Vzc2VzOgogICAgdGFza19jb21wbGV0aW9uX3Byb2Nlc3NlcyA9IFsKICAgICAgICBQcm9jZXNzKAogICAgICAgICAgICB0YXJnZXQ9X211bHRpcHJvY2Vzc2luZ19jb21wbGV0ZV90YXNrcywKICAgICAgICAgICAga3dhcmdzPXsKICAgICAgICAgICAgICAgICJ2YWRfaW5pdF9rd2FyZ3MiOiB2YWRfaW5pdF9rd2FyZ3MsCiAgICAgICAgICAgICAgICAidGFza3NfcXVldWUiOiB0YXNrc19xdWV1ZSwKICAgICAgICAgICAgICAgICJyZXN1bHRzX3F1ZXVlIjogcmVzdWx0c19xdWV1ZSwKICAgICAgICAgICAgfSwKICAgICAgICApCiAgICAgICAgZm9yIF8gaW4gcmFuZ2Uobl93b3JrZXJzKQogICAgXQoKICAgICMgU3RhcnQgdGhlIG11bHRpcHJvY2Vzc2luZyBwcm9jZXNzZXM6CiAgICBmb3IgcCBpbiB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzOgogICAgICAgIHAuc3RhcnQoKQoKICAgICMgUHV0IHRoZSB0YXNrcyBpbiB0aGUgcXVldWU6CiAgICBmb3IgYXVkaW9fZmlsZSBpbiBhdWRpb19maWxlczoKICAgICAgICB0YXNrc19xdWV1ZS5wdXQodGFza19jcmVhdG9yLmNyZWF0ZV90YXNrKGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSkudG9fdHVwbGUoKSkKCiAgICAjIFB1dCB0aGUgc3RvcCBtYXJrcyBpbiB0aGUgcXVldWU6CiAgICBmb3IgXyBpbiByYW5nZShuX3dvcmtlcnMpOgogICAgICAgIHRhc2tzX3F1ZXVlLnB1dChfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSykKCiAgICAjIENvbGxlY3QgdGhlIHJlc3VsdHM6CiAgICByZXN1bHRzID0gW10KICAgIHN0b3BfbWFya3NfY291bnRlciA9IDAKICAgIHdpdGggdHFkbSgKICAgICAgICBkZXNjPWRlc2NyaXB0aW9uLAogICAgICAgIHVuaXQ9ImZpbGUiLAogICAgICAgIHRvdGFsPWxlbihhdWRpb19maWxlcyksCiAgICAgICAgZGlzYWJsZT1ub3QgdmVyYm9zZSwKICAgICkgYXMgcHJvZ3Jlc3NiYXI6CiAgICAgICAgd2hpbGUgVHJ1ZToKICAgICAgICAgICAgIyBHZXQgYSByZXN1bHQgZnJvbSB0aGUgcXVldWU6CiAgICAgICAgICAgIHJlc3VsdDogVHVwbGVbYm9vbCwgVHVwbGVbc3RyLCBsaXN0XV0gPSByZXN1bHRzX3F1ZXVlLmdldCgpCiAgICAgICAgICAgIGlmIHJlc3VsdCA9PSBfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSzoKICAgICAgICAgICAgICAgIHN0b3BfbWFya3NfY291bnRlciArPSAxCiAgICAgICAgICAgICAgICBpZiBzdG9wX21hcmtzX2NvdW50ZXIgPT0gbl93b3JrZXJzOgogICAgICAgICAgICAgICAgICAgIGJyZWFrCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAjIENvbGxlY3QgdGhlIHJlc3VsdDoKICAgICAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKHJlc3VsdCkKICAgICAgICAgICAgICAgIHByb2dyZXNzYmFyLnVwZGF0ZSgxKQoKICAgICMgV2FpdCBmb3IgdGhlIHByb2Nlc3NlcyB0byBmaW5pc2g6CiAgICBmb3IgcCBpbiB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzOgogICAgICAgIHAuam9pbigpCgogICAgcmV0dXJuIHJlc3VsdHMKCgpkZWYgX3Byb2Nlc3NfcmVzdWx0cygKICAgIHJlc3VsdHM6IExpc3RbVHVwbGVbYm9vbCwgVHVwbGVbc3RyLCBsaXN0XV1dLCB2ZXJib3NlOiBib29sCikgLT4gVHVwbGVbZGljdCwgZGljdF06CiAgICAiIiIKICAgIFByb2Nlc3MgdGhlIHJlc3VsdHMgb2YgdGhlIHRhc2tzLgoKICAgIDpwYXJhbSByZXN1bHRzOiBUaGUgcmVzdWx0cyB0byBwcm9jZXNzLgogICAgOnBhcmFtIHZlcmJvc2U6IFZlcmJvc2l0eS4KCiAgICA6cmV0dXJuczogVGhlIHByb2Nlc3NlZCByZXN1bHRzIGFzIGEgdHVwbGUgb2Ygc3VjY2Vzc2VzIGFuZCBlcnJvcnMuCiAgICAiIiIKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJTdW1tYXJpemluZyB0aGUgcmVzdWx0cy4iKQogICAgc3VjY2Vzc2VzID0ge30KICAgIGVycm9ycyA9IHt9CiAgICBmb3IgaXNfZXJyb3IsIHJlc3VsdCBpbiByZXN1bHRzOgogICAgICAgIGlmIGlzX2Vycm9yOgogICAgICAgICAgICBlcnJvcnNbcmVzdWx0WzBdXSA9IHJlc3VsdFsxXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHN1Y2Nlc3Nlc1tyZXN1bHRbMF1dID0gcmVzdWx0WzFdCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkRvbmUgKHtsZW4oc3VjY2Vzc2VzKX0ve2xlbihzdWNjZXNzZXMpICsgbGVuKGVycm9ycyl9KVxuIikKCiAgICByZXR1cm4gc3VjY2Vzc2VzLCBlcnJvcnMK + base_image: mlrun/mlrun + commands: [] + code_origin: '' + origin_filename: '' + requirements: + - torch + - torchaudio + - tqdm + - onnxruntime + entry_points: + audio_file: + name: audio_file + doc: Get the audio file of the task. + parameters: + - name: self + outputs: + - doc: The audio file of the task. + type: Path + default: '' + lineno: 43 + do_task: + name: do_task + doc: Do the task on the given speech timestamps. The task will diarize the VAD + speech timestamps into speakers. + parameters: + - name: self + - name: speech_timestamps + type: List[List[Dict[str, int]]] + doc: The speech timestamps per channel to do the task on as outputted from + the VAD. + outputs: + - default: '' + lineno: 94 + get_result: + name: get_result + doc: Get the result of the task. A tuple of the audio file name and the result. + parameters: + - name: self + outputs: + - doc: The result of the task. + default: '' + lineno: 61 + to_tuple: + name: to_tuple + doc: Convert the task to a tuple to reconstruct it later (used for multiprocessing + to pass in queue). + parameters: + - name: self + outputs: + - doc: The converted task. + default: '' + lineno: 116 + create_task: + name: create_task + doc: Create a task with the given audio file. + parameters: + - name: self + - name: audio_file + type: Path + doc: The audio file to assign to the task. + outputs: + - doc: The created task. + type: BaseTask + default: '' + lineno: 146 + from_tuple: + name: from_tuple + doc: Create a task from a tuple of the audio file name and the task kwargs. + parameters: + - name: cls + - name: task_tuple + type: Tuple[str, dict] + doc: The task tuple to create the task from. + outputs: + - doc: The created task. + type: BaseTask + default: '' + lineno: 157 + load: + name: load + doc: Load the VAD model. + parameters: + - name: self + - name: force_reload + type: bool + doc: Whether to force reload the model even if it was already loaded. Default + is True. + default: true + outputs: + - default: '' + lineno: 234 + detect_voice: + name: detect_voice + doc: "Perform voice activity detection on given audio files using the silero\ + \ VAD model -\nhttps://github.com/snakers4/silero-vad. The end result is a\ + \ dictionary with the file names as keys and their\nVAD timestamps dictionaries\ + \ as value.\n\nFor example::\n\n {\n \"file_1.wav\": [\n \ + \ {\"start\": 0, \"end\": 16000},\n {\"start\": 16000, \"end\"\ + : 32000},\n {\"start\": 32000, \"end\": 48000},\n ...\n\ + \ ],\n \"file_2.wav\": [\n {\"start\": 0, \"end\"\ + : 16000},\n {\"start\": 16000, \"end\": 32000},\n {\"\ + start\": 32000, \"end\": 48000},\n ...\n ],\n ...\n\ + \ }" + parameters: + - name: data_path + type: Union[str, Path, List[Union[str, Path]]] + doc: The path to the audio files to diarize. Can be a path to a single file, + a path to a directory or a list of paths to files. + - name: use_onnx + type: bool + doc: Whether to use ONNX for inference. Default is True. + default: true + - name: force_onnx_cpu + type: bool + doc: Whether to force ONNX to use CPU for inference. Default is True. + default: true + - name: threshold + type: float + doc: Speech threshold. Silero VAD outputs speech probabilities for each audio + chunk, probabilities ABOVE this value are considered as SPEECH. It is better + to tune this parameter for each dataset separately, but "lazy" 0.5 is pretty + good for most datasets. + default: 0.5 + - name: sampling_rate + type: int + doc: Currently, silero VAD models support 8000 and 16000 sample rates. + default: 16000 + - name: min_speech_duration_ms + type: int + doc: Final speech chunks shorter min_speech_duration_ms are thrown out. + default: 250 + - name: max_speech_duration_s + type: float + doc: Maximum duration of speech chunks in seconds. Chunks longer than `max_speech_duration_s` + will be split at the timestamp of the last silence that lasts more than + 100ms (if any), to prevent aggressive cutting. Otherwise, they will be split + aggressively just before max_speech_duration_s. + default: float('inf') + - name: min_silence_duration_ms + type: int + doc: In the end of each speech chunk wait for min_silence_duration_ms before + separating it. + default: 100 + - name: window_size_samples + type: int + doc: Audio chunks of window_size_samples size are fed to the silero VAD model. + default: 512 + - name: speech_pad_ms + type: int + doc: Final speech chunks are padded by speech_pad_ms each side. + default: 30 + - name: return_seconds + type: bool + doc: Whether return timestamps in seconds. False means to return timestamps + in samples (default - False). + default: false + - name: per_channel + type: bool + doc: Whether to return timestamps per channel (default - False). This will + run VAD on each channel separately and return a list of timestamps per channel. + default: false + - name: use_multiprocessing + type: int + doc: The number of workers to use for multiprocessing. If 0, no multiprocessing + will be used. Default is 0. + default: 0 + - name: verbose + type: bool + doc: Verbosity. + default: false + outputs: + - default: '' + lineno: 393 + diarize: + name: diarize + doc: "Perform speech diarization on given audio files using the silero VAD model\ + \ - https://github.com/snakers4/silero-vad.\nThe speech diarization is performed\ + \ per channel so that each channel in the audio belong to a different speaker.\ + \ The\nend result is a dictionary with the file names as keys and their diarization\ + \ as value. A diarization is a list\nof tuples: (start, end, speaker_label).\n\ + \nFor example::\n\n {\n \"file_1.wav\": [\n (0.0, 1.0,\ + \ \"speaker_0\"),\n (1.0, 2.0, \"speaker_1\"),\n (2.0,\ + \ 3.0, \"speaker_0\"),\n ...\n ],\n \"file_2.wav\"\ + : [\n (0.0, 1.0, \"speaker_0\"),\n (1.0, 2.0, \"speaker_1\"\ + ),\n (2.0, 3.0, \"speaker_0\"),\n ...\n ],\n\ + \ ...\n }" + parameters: + - name: data_path + type: Union[str, Path, List[Union[str, Path]]] + doc: The path to the audio files to diarize. Can be a path to a single file, + a path to a directory or a list of paths to files. + - name: use_onnx + type: bool + doc: Whether to use ONNX for inference. Default is True. + default: true + - name: force_onnx_cpu + type: bool + doc: Whether to force ONNX to use CPU for inference. Default is True. + default: true + - name: threshold + type: float + doc: Speech threshold. Silero VAD outputs speech probabilities for each audio + chunk, probabilities ABOVE this value are considered as SPEECH. It is better + to tune this parameter for each dataset separately, but "lazy" 0.5 is pretty + good for most datasets. + default: 0.5 + - name: sampling_rate + type: int + doc: Currently, silero VAD models support 8000 and 16000 sample rates. + default: 16000 + - name: min_speech_duration_ms + type: int + doc: Final speech chunks shorter min_speech_duration_ms are thrown out. + default: 250 + - name: max_speech_duration_s + type: float + doc: Maximum duration of speech chunks in seconds. Chunks longer than `max_speech_duration_s` + will be split at the timestamp of the last silence that lasts more than + 100ms (if any), to prevent aggressive cutting. Otherwise, they will be split + aggressively just before max_speech_duration_s. + default: float('inf') + - name: min_silence_duration_ms + type: int + doc: In the end of each speech chunk wait for min_silence_duration_ms before + separating it. + default: 100 + - name: window_size_samples + type: int + doc: Audio chunks of window_size_samples size are fed to the silero VAD model. + default: 512 + - name: speech_pad_ms + type: int + doc: Final speech chunks are padded by speech_pad_ms each side. + default: 30 + - name: speaker_labels + type: List[str] + doc: The speaker labels to use for the diarization. If not given, the speakers + will be named "speaker_0", "speaker_1", etc. + default: null + - name: use_multiprocessing + type: int + doc: The number of workers to use for multiprocessing. If 0, no multiprocessing + will be used. Default is 0. + default: 0 + - name: verbose + type: bool + doc: Verbosity. + default: false + outputs: + - default: '' + lineno: 517 + description: Silero VAD (Voice Activity Detection) functions. + default_handler: detect_voice + disable_auto_mount: false + clone_target_dir: '' + env: [] + priority_class_name: '' + preemption_mode: prevent + affinity: null + tolerations: null + security_context: {} +verbose: false diff --git a/silero_vad/item.yaml b/silero_vad/item.yaml new file mode 100644 index 000000000..6f85a4c7d --- /dev/null +++ b/silero_vad/item.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +categories: + - Deep Learning + - PyTorch + - Audio +description: Silero VAD (Voice Activity Detection) functions. +doc: '' +example: silero_vad.ipynb +generationDate: 2023-12-03:14-30 +hidden: false +icon: '' +labels: + author: guyl +maintainers: [] +marketplaceType: '' +mlrunVersion: 1.5.2 +name: silero_vad +platformVersion: 3.5.3 +spec: + filename: silero_vad.py + handler: detect_voice + image: mlrun/mlrun + kind: job + requirements: + - torch + - torchaudio + - tqdm + - onnxruntime +url: '' +version: 1.1.0 diff --git a/silero_vad/silero_vad.ipynb b/silero_vad/silero_vad.ipynb new file mode 100644 index 000000000..29cd7437e --- /dev/null +++ b/silero_vad/silero_vad.ipynb @@ -0,0 +1,35 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "initial_id", + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/silero_vad/silero_vad.py b/silero_vad/silero_vad.py new file mode 100644 index 000000000..a477d4ecf --- /dev/null +++ b/silero_vad/silero_vad.py @@ -0,0 +1,847 @@ +# Copyright 2024 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from multiprocessing import Process, Queue +from pathlib import Path +from types import FunctionType +from typing import Dict, List, Tuple, Type, Union + +import torch +import torchaudio +from tqdm import tqdm + + +class BaseTask: + """ + A base class for a task to complete after VAD. + """ + + def __init__(self, audio_file: Path): + """ + Initialize the base task. + + :param audio_file: The audio file assigned to the task. + """ + # Store the audio file: + self._audio_file = audio_file + + # Prepare the result: + self._result = None + + @property + def audio_file(self) -> Path: + """ + Get the audio file of the task. + + :returns: The audio file of the task. + """ + return self._audio_file + + def do_task( + self, speech_timestamps: Union[List[Dict[str, int]], List[List[Dict[str, int]]]] + ): + """ + Do the task on the given speech timestamps. The base task will simply save the speech timestamps as the result. + + :param speech_timestamps: The speech timestamps to do the task on as outputted from the VAD. + """ + self._result = speech_timestamps + + def get_result(self) -> Tuple[str, list]: + """ + Get the result of the task. A tuple of the audio file name and the result. + + :returns: The result of the task. + """ + return self._audio_file.name, self._result + + def to_tuple(self) -> Tuple[str, dict]: + """ + Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). + + :returns: The converted task. + """ + return self.__class__.__name__, {"audio_file": self._audio_file} + + +class SpeechDiarizationTask(BaseTask): + """ + A speech diarization task. The task will diarize the VAD speech timestamps into speakers. + """ + + def __init__(self, audio_file: Path, speaker_labels: List[str]): + """ + Initialize the speech diarization task. + + :param audio_file: The audio file assigned to the task. + :param speaker_labels: The speaker labels to use for the diarization. If not given, the speakers will be named + "speaker_0", "speaker_1", etc. + """ + super().__init__(audio_file=audio_file) + self._speaker_labels = speaker_labels + + def do_task(self, speech_timestamps: List[List[Dict[str, int]]]): + """ + Do the task on the given speech timestamps. The task will diarize the VAD speech timestamps into speakers. + + :param speech_timestamps: The speech timestamps per channel to do the task on as outputted from the VAD. + """ + # Get the speaker labels (set default if not given): + speaker_labels = self._speaker_labels or [ + f"speaker_{i}" for i in range(len(speech_timestamps)) + ] + + # Diarize - organize the speech timestamps into a single list of speakers and sort it by start time: + speech_diarization = [ + (speech_timestamp["start"], speech_timestamp["end"], speaker_label) + for speaker_label, channel_speech_timestamps in zip( + speaker_labels, speech_timestamps + ) + for speech_timestamp in channel_speech_timestamps + ] + speech_diarization.sort() + self._result = speech_diarization + + def to_tuple(self) -> Tuple[str, dict]: + """ + Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). + + :returns: The converted task. + """ + task_class, task_kwargs = super().to_tuple() + return task_class, {**task_kwargs, "speaker_labels": self._speaker_labels} + + +class TaskCreator: + """ + A task creator to create different tasks to run after the VAD. + """ + + #: A map from task class name to task class to use in `from_tuple`: + _MAP = { + BaseTask.__name__: BaseTask, + SpeechDiarizationTask.__name__: SpeechDiarizationTask, + } + + def __init__(self, task_type: Type[BaseTask], task_kwargs: dict = None): + """ + Initialize the task creator. + :param task_type: The task type - a `BaseTask` subclass. + :param task_kwargs: Additional keyword arguments to pass to the to be created tasks. + """ + self._task_type = task_type + self._task_kwargs = task_kwargs or {} + + def create_task(self, audio_file: Path) -> BaseTask: + """ + Create a task with the given audio file. + + :param audio_file: The audio file to assign to the task. + + :returns: The created task. + """ + return self._task_type(audio_file=audio_file, **self._task_kwargs) + + @classmethod + def from_tuple(cls, task_tuple: Tuple[str, dict]) -> BaseTask: + """ + Create a task from a tuple of the audio file name and the task kwargs. + + :param task_tuple: The task tuple to create the task from. + + :returns: The created task. + """ + task_class, task_kwargs = task_tuple + return cls._MAP[task_class](**task_kwargs) + + +class VoiceActivityDetector: + """ + A voice activity detection wrapper for the silero VAD model - https://github.com/snakers4/silero-vad. + """ + + def __init__( + self, + # Model loading kwargs: + use_onnx: bool = True, + force_onnx_cpu: bool = True, + # Detection kwargs: + threshold: float = 0.5, + sampling_rate: int = 16_000, + min_speech_duration_ms: int = 250, + max_speech_duration_s: float = float("inf"), + min_silence_duration_ms: int = 100, + window_size_samples: int = 512, + speech_pad_ms: int = 30, + return_seconds: bool = False, + per_channel: bool = False, + ): + """ + Initialize the voice activity detector. + + :param use_onnx: Whether to use ONNX for inference. Default is True. + :param force_onnx_cpu: Whether to force ONNX to use CPU for inference. Default is True. + :param threshold: Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, + probabilities ABOVE this value are considered as SPEECH. It is better to tune + this parameter for each dataset separately, but "lazy" 0.5 is pretty good for + most datasets. + :param sampling_rate: Currently, silero VAD models support 8000 and 16000 sample rates. + :param min_speech_duration_ms: Final speech chunks shorter min_speech_duration_ms are thrown out. + :param max_speech_duration_s: Maximum duration of speech chunks in seconds. Chunks longer than + `max_speech_duration_s` will be split at the timestamp of the last silence that + lasts more than 100ms (if any), to prevent aggressive cutting. Otherwise, + they will be split aggressively just before max_speech_duration_s. + :param min_silence_duration_ms: In the end of each speech chunk wait for min_silence_duration_ms before + separating it. + :param window_size_samples: Audio chunks of window_size_samples size are fed to the silero VAD model. + WARNING! Silero VAD models were trained using 512, 1024, 1536 samples for 16000 + sample rate and 256, 512, 768 samples for 8000 sample rate. Values other than + these may affect model performance! + :param speech_pad_ms: Final speech chunks are padded by speech_pad_ms each side. + :param return_seconds: Whether return timestamps in seconds. False means to return timestamps in + samples (default - False). + :param per_channel: Whether to return timestamps per channel (default - False). This will run VAD + on each channel separately and return a list of timestamps per channel. + """ + # Store configurations: + self._use_onnx = use_onnx + self._force_onnx_cpu = force_onnx_cpu + self._threshold = threshold + self._sampling_rate = sampling_rate + self._min_speech_duration_ms = min_speech_duration_ms + self._max_speech_duration_s = max_speech_duration_s + self._min_silence_duration_ms = min_silence_duration_ms + self._window_size_samples = window_size_samples + self._speech_pad_ms = speech_pad_ms + self._return_seconds = return_seconds + self._per_channel = per_channel + + # Prepare the model variables + self._model: torch.Module = None + self._get_speech_timestamps: FunctionType = None + + def load(self, force_reload: bool = True): + """ + Load the VAD model. + + :param force_reload: Whether to force reload the model even if it was already loaded. Default is True. + """ + model, utils = torch.hub.load( + repo_or_dir="snakers4/silero-vad", + model="silero_vad", + force_reload=force_reload, + onnx=self._use_onnx, + force_onnx_cpu=self._force_onnx_cpu, + ) + self._model = model + ( + self._get_speech_timestamps, + _, # save_audio, + _, # read_audio, + _, # VADIterator, + _, # collect_chunks + ) = utils + + def detect_voice( + self, + audio_file: Path, + ) -> Union[List[Dict[str, int]], List[List[Dict[str, int]]]]: + """ + Infer the audio through the VAD model and return the speech timestamps. + + :param audio_file: The audio file to infer. + + :returns: The speech timestamps in the audio. A list of timestamps where each timestamp is a dictionary with the + following keys: + + * "start": The start sample index of the speech in the audio. + * "end": The end sample index of the speech in the audio. + + If `per_channel` is True, a list of timestamps per channel will be returned. + """ + # Cast to a numpy array: + audio = self._read_audio(audio_file) + + # Detect speech: + if not self._per_channel: + return self._get_speech_timestamps( + audio, + self._model, + threshold=self._threshold, + min_speech_duration_ms=self._min_speech_duration_ms, + max_speech_duration_s=self._max_speech_duration_s, + min_silence_duration_ms=self._min_silence_duration_ms, + speech_pad_ms=self._speech_pad_ms, + sampling_rate=self._sampling_rate, + window_size_samples=self._window_size_samples, + return_seconds=self._return_seconds, + ) + + # Per channel: + speech_timestamps = [] + for channel in audio: + speech_timestamps.append( + self._get_speech_timestamps( + channel, + self._model, + threshold=self._threshold, + min_speech_duration_ms=self._min_speech_duration_ms, + max_speech_duration_s=self._max_speech_duration_s, + min_silence_duration_ms=self._min_silence_duration_ms, + speech_pad_ms=self._speech_pad_ms, + sampling_rate=self._sampling_rate, + window_size_samples=self._window_size_samples, + return_seconds=self._return_seconds, + ) + ) + + return speech_timestamps + + def _read_audio( + self, + path: Path, + ) -> torch.Tensor: + """ + Read the audio from the given path and return it as a tensor. + + :param path: The path to the audio file. + + :returns: The audio as a tensor. + """ + # Read the audio: + audio, sampling_rate = torchaudio.load(str(path)) + + # Check if the audio is stereo and if so, convert it to mono (only if not per channel): + if audio.size(0) > 1 and not self._per_channel: + audio = audio.mean(dim=0, keepdim=True) + + # Resample the audio if needed: + if sampling_rate != self._sampling_rate: + transform = torchaudio.transforms.Resample( + orig_freq=sampling_rate, new_freq=self._sampling_rate + ) + audio = transform(audio) + + # Return the audio (squeeze if not per channel): + return audio if self._per_channel else audio.squeeze(0) + + +#: The value to send into multiprocessing queues to stop the process: +_MULTIPROCESSING_STOP_MARK = "STOP" + + +def _multiprocessing_complete_tasks( + vad_init_kwargs: dict, tasks_queue: Queue, results_queue: Queue +): + """ + Complete the tasks in the given queue and put the results in the given results queue. The function will stop when + the given tasks queue will receive the stop mark. It is aimed to be used with multiprocessing as a process. + + :param vad_init_kwargs: The VAD initialization kwargs. + :param tasks_queue: A queue to get the tasks from. + :param results_queue: A queue to put the results in. + """ + # Initialize and load the VAD: + vad = VoiceActivityDetector(**vad_init_kwargs) + vad.load(force_reload=False) + + # Start listening to the tasks queue: + while True: + # Get the task: + task: Tuple[str, dict] = tasks_queue.get() + if task == _MULTIPROCESSING_STOP_MARK: + break + try: + # Create the task: + task = TaskCreator.from_tuple(task_tuple=task) + # Run the file through the VAD: + speech_timestamps = vad.detect_voice(audio_file=task.audio_file) + # Complete the task: + task.do_task(speech_timestamps=speech_timestamps) + # Build the result: + result = (False, task.get_result()) + except Exception as exception: + # Build the error: + result = (True, (task.audio_file.name, str(exception))) + # Collect the result / error: + results_queue.put(result) + + # Mark the end of the tasks: + results_queue.put(_MULTIPROCESSING_STOP_MARK) + + +# Get the global logger: +try: + import mlrun + + _LOGGER = mlrun.get_or_create_ctx("silero_vad").logger +except ModuleNotFoundError: + _LOGGER = logging.getLogger() + + +def detect_voice( + # Input kwargs: + data_path: Union[str, Path, List[Union[str, Path]]], + # Model loading kwargs: + use_onnx: bool = True, + force_onnx_cpu: bool = True, + # Detection kwargs: + threshold: float = 0.5, + sampling_rate: int = 16_000, + min_speech_duration_ms: int = 250, + max_speech_duration_s: float = float("inf"), + min_silence_duration_ms: int = 100, + window_size_samples: int = 512, + speech_pad_ms: int = 30, + return_seconds: bool = False, + per_channel: bool = False, + # Other kwargs: + use_multiprocessing: int = 0, + verbose: bool = False, +): + """ + Perform voice activity detection on given audio files using the silero VAD model - + https://github.com/snakers4/silero-vad. The end result is a dictionary with the file names as keys and their + VAD timestamps dictionaries as value. + + For example:: + + { + "file_1.wav": [ + {"start": 0, "end": 16000}, + {"start": 16000, "end": 32000}, + {"start": 32000, "end": 48000}, + ... + ], + "file_2.wav": [ + {"start": 0, "end": 16000}, + {"start": 16000, "end": 32000}, + {"start": 32000, "end": 48000}, + ... + ], + ... + } + + + :param data_path: The path to the audio files to diarize. Can be a path to a single file, a path to a + directory or a list of paths to files. + :param use_onnx: Whether to use ONNX for inference. Default is True. + :param force_onnx_cpu: Whether to force ONNX to use CPU for inference. Default is True. + :param threshold: Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, + probabilities ABOVE this value are considered as SPEECH. It is better to tune + this parameter for each dataset separately, but "lazy" 0.5 is pretty good for + most datasets. + :param sampling_rate: Currently, silero VAD models support 8000 and 16000 sample rates. + :param min_speech_duration_ms: Final speech chunks shorter min_speech_duration_ms are thrown out. + :param max_speech_duration_s: Maximum duration of speech chunks in seconds. Chunks longer than + `max_speech_duration_s` will be split at the timestamp of the last silence that + lasts more than 100ms (if any), to prevent aggressive cutting. Otherwise, they will + be split aggressively just before max_speech_duration_s. + :param min_silence_duration_ms: In the end of each speech chunk wait for min_silence_duration_ms before separating + it. + :param window_size_samples: Audio chunks of window_size_samples size are fed to the silero VAD model. + + WARNING! Silero VAD models were trained using 512, 1024, 1536 samples for 16000 + sample rate and 256, 512, 768 samples for 8000 sample rate. Values other than + these may affect model performance! + :param speech_pad_ms: Final speech chunks are padded by speech_pad_ms each side. + :param return_seconds: Whether return timestamps in seconds. False means to return timestamps in samples + (default - False). + :param per_channel: Whether to return timestamps per channel (default - False). This will run VAD on + each channel separately and return a list of timestamps per channel. + :param use_multiprocessing: The number of workers to use for multiprocessing. If 0, no multiprocessing will + be used. Default is 0. + :param verbose: Verbosity. + """ + global _LOGGER + + # Get the input audio files to transcribe: + if verbose: + _LOGGER.info("Collecting audio files.") + audio_files = _get_audio_files(data_path=data_path) + if verbose: + _LOGGER.info(f"Collected {len(audio_files)} audio files.") + + # Initialize the transcription pipeline: + vad_init_kwargs = { + "use_onnx": use_onnx, + "force_onnx_cpu": force_onnx_cpu, + "threshold": threshold, + "sampling_rate": sampling_rate, + "min_speech_duration_ms": min_speech_duration_ms, + "max_speech_duration_s": max_speech_duration_s, + "min_silence_duration_ms": min_silence_duration_ms, + "window_size_samples": window_size_samples, + "speech_pad_ms": speech_pad_ms, + "return_seconds": return_seconds, + "per_channel": per_channel, + } + + # Create the task creator: + task_creator = TaskCreator(task_type=BaseTask) + + # Run the transcription: + if use_multiprocessing: + results = _parallel_run( + n_workers=use_multiprocessing, + audio_files=audio_files, + description="Detecting voice", + vad_init_kwargs=vad_init_kwargs, + task_creator=task_creator, + verbose=verbose, + ) + else: + results = _run( + audio_files=audio_files, + description="Detecting voice", + vad_init_kwargs=vad_init_kwargs, + task_creator=task_creator, + verbose=verbose, + ) + + # Process the results: + return _process_results(results=results, verbose=verbose) + + +def diarize( + # Input / Output kwargs: + data_path: Union[str, Path, List[Union[str, Path]]], + # Model loading kwargs: + use_onnx: bool = True, + force_onnx_cpu: bool = True, + # Detection kwargs: + threshold: float = 0.5, + sampling_rate: int = 16_000, + min_speech_duration_ms: int = 250, + max_speech_duration_s: float = float("inf"), + min_silence_duration_ms: int = 100, + window_size_samples: int = 512, + speech_pad_ms: int = 30, + # Diarization kwargs: + speaker_labels: List[str] = None, + # Other kwargs: + use_multiprocessing: int = 0, + verbose: bool = False, +): + """ + Perform speech diarization on given audio files using the silero VAD model - https://github.com/snakers4/silero-vad. + The speech diarization is performed per channel so that each channel in the audio belong to a different speaker. The + end result is a dictionary with the file names as keys and their diarization as value. A diarization is a list + of tuples: (start, end, speaker_label). + + For example:: + + { + "file_1.wav": [ + (0.0, 1.0, "speaker_0"), + (1.0, 2.0, "speaker_1"), + (2.0, 3.0, "speaker_0"), + ... + ], + "file_2.wav": [ + (0.0, 1.0, "speaker_0"), + (1.0, 2.0, "speaker_1"), + (2.0, 3.0, "speaker_0"), + ... + ], + ... + } + + + :param data_path: The path to the audio files to diarize. Can be a path to a single file, a path to a + directory or a list of paths to files. + :param use_onnx: Whether to use ONNX for inference. Default is True. + :param force_onnx_cpu: Whether to force ONNX to use CPU for inference. Default is True. + :param threshold: Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, + probabilities ABOVE this value are considered as SPEECH. It is better to tune + this parameter for each dataset separately, but "lazy" 0.5 is pretty good for + most datasets. + :param sampling_rate: Currently, silero VAD models support 8000 and 16000 sample rates. + :param min_speech_duration_ms: Final speech chunks shorter min_speech_duration_ms are thrown out. + :param max_speech_duration_s: Maximum duration of speech chunks in seconds. Chunks longer than + `max_speech_duration_s` will be split at the timestamp of the last silence that + lasts more than 100ms (if any), to prevent aggressive cutting. Otherwise, they will + be split aggressively just before max_speech_duration_s. + :param min_silence_duration_ms: In the end of each speech chunk wait for min_silence_duration_ms before separating + it. + :param window_size_samples: Audio chunks of window_size_samples size are fed to the silero VAD model. + + WARNING! Silero VAD models were trained using 512, 1024, 1536 samples for 16000 + sample rate and 256, 512, 768 samples for 8000 sample rate. Values other than + these may affect model performance! + :param speech_pad_ms: Final speech chunks are padded by speech_pad_ms each side. + :param speaker_labels: The speaker labels to use for the diarization. If not given, the speakers will be + named "speaker_0", "speaker_1", etc. + :param use_multiprocessing: The number of workers to use for multiprocessing. If 0, no multiprocessing will + be used. Default is 0. + :param verbose: Verbosity. + """ + global _LOGGER + + # Get the input audio files to transcribe: + if verbose: + _LOGGER.info("Collecting audio files.") + audio_files = _get_audio_files(data_path=data_path) + if verbose: + _LOGGER.info(f"Collected {len(audio_files)} audio files.") + + # Initialize the transcription pipeline: + vad_init_kwargs = { + "use_onnx": use_onnx, + "force_onnx_cpu": force_onnx_cpu, + "threshold": threshold, + "sampling_rate": sampling_rate, + "min_speech_duration_ms": min_speech_duration_ms, + "max_speech_duration_s": max_speech_duration_s, + "min_silence_duration_ms": min_silence_duration_ms, + "window_size_samples": window_size_samples, + "speech_pad_ms": speech_pad_ms, + "return_seconds": True, + "per_channel": True, + } + + # Create the task creator: + task_creator = TaskCreator( + task_type=SpeechDiarizationTask, task_kwargs={"speaker_labels": speaker_labels} + ) + + # Run the transcription: + if use_multiprocessing: + results = _parallel_run( + n_workers=use_multiprocessing, + audio_files=audio_files, + description="Diarizing", + vad_init_kwargs=vad_init_kwargs, + task_creator=task_creator, + verbose=verbose, + ) + else: + results = _run( + audio_files=audio_files, + description="Diarizing", + vad_init_kwargs=vad_init_kwargs, + task_creator=task_creator, + verbose=verbose, + ) + + # Process the results: + return _process_results(results=results, verbose=verbose) + + +def _get_audio_files( + data_path: Union[Path, str, list], +) -> List[Path]: + """ + Get the audio files from the data path. If a path to a directory is given, all files in the directory will be + collected. + + :param data_path: The data path to collect the audio files from. + + :returns: The audio files list. + """ + # Check if given a list of paths: + if isinstance(data_path, list): + audio_files = [] + for path in data_path: + audio_files.extend(_get_audio_files(data_path=path)) + return audio_files + + # Check if given a single string path to cast it to a `pathlib.Path`: + if isinstance(data_path, str): + data_path = Path(data_path).absolute() + + # Check if the path is of a directory or a file: + if data_path.is_dir(): + # Get all files inside the directory: + audio_files = list(data_path.glob("*.*")) + elif data_path.is_file(): + audio_files = [data_path] + else: + raise ValueError( + f"Unrecognized data path. The parameter `data_path` must be a valid path to either a directory path or a " + f"file. Given: {str(data_path)} " + ) + + return audio_files + + +def _run( + audio_files: List[Path], + description: str, + vad_init_kwargs: dict, + task_creator: TaskCreator, + verbose: bool, +) -> List[Tuple[bool, Tuple[str, list]]]: + """ + Load a VAD and use it to complete the tasks that will be created on the provided files using the given task creator. + + :param audio_files: The audio files to use. + :param description: The description to use for the progress bar. + :param vad_init_kwargs: The VAD initialization keyword arguments. + :param task_creator: The task creator to use to create the tasks. + :param verbose: Verbosity. + + :returns: The collected results. + """ + # Load the VAD: + vad = VoiceActivityDetector(**vad_init_kwargs) + if verbose: + _LOGGER.info(f"Loading the VAD model.") + vad.load() + if verbose: + _LOGGER.info("VAD model loaded.") + + # Run the VAD on the audio files and collect the results: + results = [] + for audio_file in tqdm( + audio_files, + desc=description, + unit="file", + total=len(audio_files), + disable=not verbose, + ): + try: + # Create the task: + task = task_creator.create_task(audio_file=audio_file) + # Run the file through the VAD: + speech_timestamps = vad.detect_voice(audio_file=audio_file) + # Complete the task: + task.do_task(speech_timestamps=speech_timestamps) + # Collect the result: + results.append((False, task.get_result())) + except Exception as exception: + # Collect the error: + results.append((True, (audio_file.name, str(exception)))) + + return results + + +def _parallel_run( + n_workers: int, + audio_files: List[Path], + description: str, + vad_init_kwargs: dict, + task_creator: TaskCreator, + verbose: bool, +) -> List[Tuple[bool, Tuple[str, list]]]: + """ + Run multiple VAD workers with multiprocessing to complete the tasks that will be created on the provided files using + the given task creator. + + :param n_workers: The number of workers to use. + :param audio_files: The audio files to use. + :param description: The description to use for the progress bar. + :param vad_init_kwargs: The VAD initialization keyword arguments. + :param task_creator: The task creator to use to create the tasks. + :param verbose: Verbosity. + + :returns: The collected results. + """ + # Load the VAD (download once, and it will be loaded then per process later on): + if verbose: + _LOGGER.info(f"Loading the VAD model.") + vad = VoiceActivityDetector(**vad_init_kwargs) + vad.load() + if verbose: + _LOGGER.info("VAD model loaded.") + + # Check the number of workers: + if n_workers > len(audio_files): + _LOGGER.warning( + f"The number of workers ({n_workers}) is larger than the number of audio files ({len(audio_files)}). " + f"Setting the number of workers to {len(audio_files)}." + ) + n_workers = len(audio_files) + + # Initialize the multiprocessing queues: + tasks_queue = Queue() + results_queue = Queue() + + # Initialize the multiprocessing processes: + task_completion_processes = [ + Process( + target=_multiprocessing_complete_tasks, + kwargs={ + "vad_init_kwargs": vad_init_kwargs, + "tasks_queue": tasks_queue, + "results_queue": results_queue, + }, + ) + for _ in range(n_workers) + ] + + # Start the multiprocessing processes: + for p in task_completion_processes: + p.start() + + # Put the tasks in the queue: + for audio_file in audio_files: + tasks_queue.put(task_creator.create_task(audio_file=audio_file).to_tuple()) + + # Put the stop marks in the queue: + for _ in range(n_workers): + tasks_queue.put(_MULTIPROCESSING_STOP_MARK) + + # Collect the results: + results = [] + stop_marks_counter = 0 + with tqdm( + desc=description, + unit="file", + total=len(audio_files), + disable=not verbose, + ) as progressbar: + while True: + # Get a result from the queue: + result: Tuple[bool, Tuple[str, list]] = results_queue.get() + if result == _MULTIPROCESSING_STOP_MARK: + stop_marks_counter += 1 + if stop_marks_counter == n_workers: + break + else: + # Collect the result: + results.append(result) + progressbar.update(1) + + # Wait for the processes to finish: + for p in task_completion_processes: + p.join() + + return results + + +def _process_results( + results: List[Tuple[bool, Tuple[str, list]]], verbose: bool +) -> Tuple[dict, dict]: + """ + Process the results of the tasks. + + :param results: The results to process. + :param verbose: Verbosity. + + :returns: The processed results as a tuple of successes and errors. + """ + if verbose: + _LOGGER.info("Summarizing the results.") + successes = {} + errors = {} + for is_error, result in results: + if is_error: + errors[result[0]] = result[1] + else: + successes[result[0]] = result[1] + if verbose: + _LOGGER.info(f"Done ({len(successes)}/{len(successes) + len(errors)})\n") + + return successes, errors diff --git a/silero_vad/test_silero_vad.py b/silero_vad/test_silero_vad.py new file mode 100644 index 000000000..d46471a57 --- /dev/null +++ b/silero_vad/test_silero_vad.py @@ -0,0 +1,44 @@ +import os +import tempfile + +import mlrun +import pytest + + +@pytest.fixture() +def setup_test(): + with tempfile.TemporaryDirectory() as artifact_path: + project = mlrun.get_or_create_project(name="default", context=artifact_path) + func = project.set_function( + func=os.path.abspath("./function.yaml"), + name="silero-vad", + image="mlrun/mlrun", + ) + yield func, artifact_path + + +def test_detect_voice(setup_test): + silero_vad_function, artifact_path = setup_test + run = silero_vad_function.run( + handler="detect_voice", + inputs={"data_path": "./assets"}, + returns=["vad_outputs: file", "errors: file"], + artifact_path=artifact_path, + local=True, + ) + assert run.outputs["vad_outputs"] + + +def test_diarize(setup_test): + silero_vad_function, artifact_path = setup_test + run = silero_vad_function.run( + handler="diarize", + inputs={"data_path": "./assets"}, + params={ + "speakers_labels": ["Agent", "Client"], + }, + returns=["speech_diarization: file", "errors: file"], + artifact_path=artifact_path, + local=True, + ) + assert run.outputs["speech_diarization"] diff --git a/speech_diarization/function.yaml b/speech_diarization/function.yaml deleted file mode 100644 index 03b0a78d5..000000000 --- a/speech_diarization/function.yaml +++ /dev/null @@ -1,143 +0,0 @@ -kind: job -metadata: - name: speech-diarization - tag: '' - hash: 2486500a2579a422fb586752aadc02a58427f60f - project: '' - labels: - author: guyl - categories: - - Utilities - - Machine Learning -spec: - command: '' - args: [] - image: mlrun/mlrun - build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGhlYXBxCmltcG9ydCBsb2dnaW5nCmltcG9ydCBvcGVyYXRvcgppbXBvcnQgb3MKaW1wb3J0IHBhdGhsaWIKZnJvbSBmdW5jdG9vbHMgaW1wb3J0IHJlZHVjZSwgd3JhcHMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdCwgTGlzdCwgVHVwbGUsIFVuaW9uCgppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBweWFubm90ZS5hdWRpbwppbXBvcnQgcHlhbm5vdGUuY29yZQppbXBvcnQgdG9yY2gKaW1wb3J0IHRvcmNoYXVkaW8KZnJvbSB0cWRtIGltcG9ydCB0cWRtCgojIEdldCB0aGUgZ2xvYmFsIGxvZ2dlcjoKX0xPR0dFUiA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCkKCgpkZWYgX2NoZWNrX21scnVuX2FuZF9vcGVuX21waSgpIC0+IFR1cGxlWyJtbHJ1bi5NTENsaWVudEN0eCIsICJtcGk0cHkuTVBJLkludHJhY29tbSJdOgogICAgaXNfbXBpID0gRmFsc2UKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KCiAgICAgICAgY29udGV4dCA9IG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KG5hbWU9Im1scnVuIikKICAgICAgICBpc19tcGkgPSBjb250ZXh0LmxhYmVscy5nZXQoImtpbmQiLCAiam9iIikgPT0gIm1waWpvYiIKCiAgICAgICAgaWYgaXNfbXBpOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBmcm9tIG1waTRweSBpbXBvcnQgTVBJCgogICAgICAgICAgICAgICAgcmV0dXJuIGNvbnRleHQsIE1QSS5DT01NX1dPUkxECiAgICAgICAgICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yIGFzIG1waTRweV9ub3RfZm91bmQ6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgICAgICAgICAiVG8gZGlzdHJpYnV0ZSB0aGUgZnVuY3Rpb24gdXNpbmcgTUxSdW4ncyAnbXBpam9iJyB5b3UgbmVlZCB0byBoYXZlIGBtcGk0cHlgIHBhY2thZ2UgaW4geW91ciAiCiAgICAgICAgICAgICAgICAgICAgImludGVycHJldGVyLiBQbGVhc2UgcnVuIGBwaXAgaW5zdGFsbCBtcGk0cHlgIGFuZCBtYWtlIHN1cmUgeW91IGhhdmUgb3Blbi1tcGkuIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgcmFpc2UgbXBpNHB5X25vdF9mb3VuZAogICAgZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3IgYXMgbW9kdWxlX25vdF9mb3VuZDoKICAgICAgICBpZiBpc19tcGk6CiAgICAgICAgICAgIHJhaXNlIG1vZHVsZV9ub3RfZm91bmQKICAgIHJldHVybiBOb25lLCBOb25lCgoKZGVmIG9wZW5fbXBpX2hhbmRsZXIoCiAgICB3b3JrZXJfaW5wdXRzOiBMaXN0W3N0cl0sIHJvb3Rfd29ya2VyX2lucHV0czogRGljdFtzdHIsIEFueV0gPSBOb25lCik6CiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgQ2hlY2sgZm9yIE1MUnVuIGFuZCBPcGVuTVBJIGF2YWlsYWJpbGl0eToKICAgIGNvbnRleHQsIGNvbW0gPSBfY2hlY2tfbWxydW5fYW5kX29wZW5fbXBpKCkKCiAgICAjIENoZWNrIGlmIE1MUnVuIGlzIGF2YWlsYWJsZSwgc2V0IHRoZSBnbG9iYWwgbG9nZ2VyIHRvIE1MUnVuJ3M6CiAgICBpZiBjb250ZXh0OgogICAgICAgIF9MT0dHRVIgPSBjb250ZXh0LmxvZ2dlcgoKICAgIGRlZiBkZWNvcmF0b3IoaGFuZGxlcik6CiAgICAgICAgaWYgY29tbSBpcyBOb25lIG9yIGNvbW0uR2V0X3NpemUoKSA9PSAxOgogICAgICAgICAgICByZXR1cm4gaGFuZGxlcgoKICAgICAgICBAd3JhcHMoaGFuZGxlcikKICAgICAgICBkZWYgd3JhcHBlcigqKmt3YXJncyk6CiAgICAgICAgICAgICMgR2V0IHRoZSBvcGVuIG1waSBlbnZpcm9ubWVudCBwcm9wZXJ0aWVzOgogICAgICAgICAgICBzaXplID0gY29tbS5HZXRfc2l6ZSgpCiAgICAgICAgICAgIHJhbmsgPSBjb21tLkdldF9yYW5rKCkKCiAgICAgICAgICAgICMgR2l2ZSB0aGUgY29ycmVjdCBjaHVuayBvZiB0aGUgd29ya2VycyBpbnB1dHM6CiAgICAgICAgICAgIGZvciB3b3JrZXJfaW5wdXQgaW4gd29ya2VyX2lucHV0czoKICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0ga3dhcmdzW3dvcmtlcl9pbnB1dF0KICAgICAgICAgICAgICAgIGlmIGlucHV0X2FyZ3VtZW50IGlzIE5vbmU6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIHN0cik6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBfZ2V0X2F1ZGlvX2ZpbGVzKAogICAgICAgICAgICAgICAgICAgICAgICBkYXRhX3BhdGg9cGF0aGxpYi5QYXRoKGlucHV0X2FyZ3VtZW50KS5hYnNvbHV0ZSgpCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgbGVuKGlucHV0X2FyZ3VtZW50KSA8IHNpemU6CiAgICAgICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAgICAgZiJDYW5ub3Qgc3BsaXQgdGhlIGlucHV0ICd7d29ya2VyX2lucHV0fScgb2YgbGVuZ3RoIHtsZW4oaW5wdXRfYXJndW1lbnQpfSB0byB7c2l6ZX0gd29ya2Vycy4gIgogICAgICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSByZWR1Y2UgdGhlIGFtb3VudCBvZiB3b3JrZXJzIGZvciB0aGlzIGlucHV0LiIKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBldmVuX2NodW5rX3NpemUgPSBsZW4oaW5wdXRfYXJndW1lbnQpIC8vIHNpemUKICAgICAgICAgICAgICAgIGNodW5rX3N0YXJ0ID0gcmFuayAqIGV2ZW5fY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgY2h1bmtfZW5kID0gKAogICAgICAgICAgICAgICAgICAgIChyYW5rICsgMSkgKiBldmVuX2NodW5rX3NpemUKICAgICAgICAgICAgICAgICAgICBpZiByYW5rICsgMSA8IHNpemUKICAgICAgICAgICAgICAgICAgICBlbHNlIGxlbihpbnB1dF9hcmd1bWVudCkKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSYW5rICN7cmFua306IFByb2Nlc3NpbmcgaW5wdXQgY2h1bmsgb2YgJ3t3b3JrZXJfaW5wdXR9JyAiCiAgICAgICAgICAgICAgICAgICAgZiJmcm9tIGluZGV4IHtjaHVua19zdGFydH0gdG8ge2NodW5rX2VuZH0uIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgbGlzdCk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudFtjaHVua19zdGFydDpjaHVua19lbmRdCiAgICAgICAgICAgICAgICBlbGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIHBkLkRhdGFGcmFtZSk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudC5pbG9jW2NodW5rX3N0YXJ0OmNodW5rX2VuZDosIDpdCiAgICAgICAgICAgICAgICBrd2FyZ3Nbd29ya2VyX2lucHV0XSA9IGlucHV0X2FyZ3VtZW50CgogICAgICAgICAgICAjIFNldCB0aGUgcm9vdCB3b3JrZXIgb25seSBhcmd1bWVudHM6CiAgICAgICAgICAgIGlmIHJhbmsgPT0gMCBhbmQgcm9vdF93b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAga3dhcmdzLnVwZGF0ZShyb290X3dvcmtlcl9pbnB1dHMpCgogICAgICAgICAgICAjIFJ1biB0aGUgd29ya2VyOgogICAgICAgICAgICBvdXRwdXQgPSBoYW5kbGVyKCoqa3dhcmdzKQoKICAgICAgICAgICAgIyBTZW5kIHRoZSBvdXRwdXQgdG8gdGhlIHJvb3QgcmFuayAocmFuayAjMCk6CiAgICAgICAgICAgIG91dHB1dCA9IGNvbW0uZ2F0aGVyKG91dHB1dCwgcm9vdD0wKQogICAgICAgICAgICBpZiByYW5rID09IDA6CiAgICAgICAgICAgICAgICAjIEpvaW4gdGhlIG91dHB1dHM6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJDb2xsZWN0aW5nIGRhdGEgZnJvbSB3b3JrZXJzIHRvIHJvb3Qgd29ya2VyLiIpCiAgICAgICAgICAgICAgICBkaWFyaXphdGlvbl9kaWN0aW9uYXJ5ID0gcmVkdWNlKAogICAgICAgICAgICAgICAgICAgIG9wZXJhdG9yLmlvciwgW2RpYSBmb3IgZGlhLCBfIGluIG91dHB1dF0sIHt9CiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBlcnJvcnNfZGljdGlvbmFyeSA9IHJlZHVjZShvcGVyYXRvci5pb3IsIFtlcnIgZm9yIF8sIGVyciBpbiBvdXRwdXRdLCB7fSkKICAgICAgICAgICAgICAgIHJldHVybiBkaWFyaXphdGlvbl9kaWN0aW9uYXJ5LCBlcnJvcnNfZGljdGlvbmFyeQogICAgICAgICAgICByZXR1cm4gTm9uZQoKICAgICAgICByZXR1cm4gd3JhcHBlcgoKICAgIHJldHVybiBkZWNvcmF0b3IKCgpAb3Blbl9tcGlfaGFuZGxlcih3b3JrZXJfaW5wdXRzPVsiZGF0YV9wYXRoIl0sIHJvb3Rfd29ya2VyX2lucHV0cz17InZlcmJvc2UiOiBUcnVlfSkKZGVmIGRpYXJpemUoCiAgICBkYXRhX3BhdGg6IFVuaW9uW3N0ciwgTGlzdFtzdHJdXSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJweWFubm90ZS9zcGVha2VyLWRpYXJpemF0aW9uLTMuMCIsCiAgICBhY2Nlc3NfdG9rZW46IHN0ciA9IE5vbmUsCiAgICBkZXZpY2U6IHN0ciA9IE5vbmUsCiAgICBzcGVha2Vyc19sYWJlbHM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBzcGVha2VyX3ByZWZpeDogc3RyID0gInNwZWFrZXJfIiwKICAgIHNlcGFyYXRlX2J5X2NoYW5uZWxzOiBib29sID0gRmFsc2UsCiAgICBtaW5pbXVtX3NwZWFrZXJzOiBpbnQgPSBOb25lLAogICAgbWF4aW11bV9zcGVha2VyczogaW50ID0gTm9uZSwKICAgIHZlcmJvc2U6IGJvb2wgPSBGYWxzZSwKKSAtPiBUdXBsZVtEaWN0W3N0ciwgTGlzdFtUdXBsZVtmbG9hdCwgZmxvYXQsIHN0cl1dXSwgRGljdFtzdHIsIHN0cl1dOgogICAgIiIiCiAgICBQZXJmb3JtIHNwZWVjaCBkaWFyaXphdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyBweWFubm90ZS1hdWRpbyAoaHR0cHM6Ly9naXRodWIuY29tL3B5YW5ub3RlL3B5YW5ub3RlLWF1ZGlvKS4KICAgIFRoZSBlbmQgcmVzdWx0IGlzIGEgZGljdGlvbmFyeSB3aXRoIHRoZSBmaWxlIG5hbWVzIGFzIGtleXMgYW5kIHRoZWlyIGRpYXJpemF0aW9uIGFzIHZhbHVlLiBBIGRpYXJpemF0aW9uIGlzIGEgbGlzdAogICAgb2YgdHVwbGVzOiAoc3RhcnQsIGVuZCwgc3BlYWtlcl9sYWJlbCkuCgogICAgVG8gdXNlIHRoZSBgcHlhbm5vdGUuYXVkaW9gIG1vZGVscyB5b3UgbXVzdCBwYXNzIGEgSHVnZ2luZ2ZhY2UgdG9rZW4gYW5kIGdldCBhY2Nlc3MgdG8gdGhlIHJlcXVpcmVkIG1vZGVscy4gVGhlCiAgICB0b2tlbiBjYW4gYmUgcGFzc2VkIGluIG9uZSBvZiB0aGUgZm9sbG93aW5nIG9wdGlvbnM6CgogICAgKiBVc2UgdGhlIHBhcmFtZXRlciBgYWNjZXNzX3Rva2VuYC4KICAgICogU2V0IGFuIGVudmlyb25tZW50IHZhcmlhYmxlIG5hbWVkICJIVUdHSU5HX0ZBQ0VfSFVCX1RPS0VOIi4KICAgICogSWYgdXNpbmcgTUxSdW4sIHlvdSBjYW4gcGFzcyBpdCBhcyBhIHNlY3JldCBuYW1lZCAiSFVHR0lOR19GQUNFX0hVQl9UT0tFTiIuCgogICAgVG8gZ2V0IGFjY2VzcyB0byB0aGUgbW9kZWxzIG9uIEh1Z2dpbmdmYWNlLCB2aXNpdCB0aGVpciBwYWdlLiBGb3IgZXhhbXBsZSwgdG8gdXNlIHRoZSBkZWZhdWx0IGRpYXJpemF0aW9uIG1vZGVsIHNldAogICAgaW4gdGhpcyBmdW5jdGlvbiAoInB5YW5ub3RlL3NwZWFrZXItZGlhcml6YXRpb24tMy4wIiksIHlvdSBuZWVkIGFjY2VzcyBmb3IgdGhlc2UgdHdvIG1vZGVsczoKCiAgICAqIGh0dHBzOi8vaHVnZ2luZ2ZhY2UuY28vcHlhbm5vdGUvc2VnbWVudGF0aW9uLTMuMAogICAgKiBodHRwczovL2h1Z2dpbmdmYWNlLmNvL3B5YW5ub3RlL3NwZWFrZXItZGlhcml6YXRpb24tMy4wCgogICAgTm90ZTogVG8gY29udHJvbCB0aGUgcmVjb2duaXplZCBzcGVha2VycyBpbiB0aGUgZGlhcml6YXRpb24gb3V0cHV0IHlvdSBjYW4gY2hvb3NlIG9uZSBvZiB0aGUgZm9sbG93aW5nIG1ldGhvZHM6CgogICAgKiBGb3IgYSBrbm93biBzcGVha2VycyBhbW91bnQsIHlvdSBtYXkgc2V0IHNwZWFrZXIgbGFiZWxzIHZpYSB0aGUgYHNwZWFrZXJzX2xhYmVsc2AgcGFyYW1ldGVyIHRoYXQgd2lsbCBiZSB1c2VkIGluCiAgICAgIHRoZSBvcmRlciBvZiBzcGVha2luZyBpbiB0aGUgYXVkaW8gKGZpcnN0IHBlcnNvbiBzcGVha2luZyBiZSB0aGUgZmlyc3QgbGFiZWwgaW4gdGhlIGxpc3QpLiBJbiBhZGRpdGlvbiwgeW91IGNhbiBkbwogICAgICBkaWFyaXphdGlvbiBwZXIgY2hhbm5lbCAoc2V0dGluZyB0aGUgcGFyYW1ldGVyIGBzZXBhcmF0ZV9ieV9jaGFubmVsc2AgdG8gVHJ1ZSkuIEVhY2ggbGFiZWwgd2lsbCBiZSBhc3NpZ25lZCB0byBhCiAgICAgIHNwZWNpZmljIGNoYW5uZWwgYnkgb3JkZXIgKGZpcnN0IGxhYmVsIHRvIGNoYW5uZWwgMCwgc2Vjb25kIGxhYmVsIHRvIGNoYW5uZWwgMSBhbmQgc28gb24pLiBOb3RpY2UsIHRoaXMgd2lsbAogICAgICBpbmNyZWFzZSBydW50aW1lLgogICAgKiBGb3IgdW5rbm93biBzcGVha2VycyBhbW91bnQsIHlvdSBjYW4gc2V0IHRoZSBgc3BlYWtlcl9wcmVmaXhgIHBhcmFtZXRlciB0byBhZGQgYSBwcmVmaXggZm9yIGVhY2ggc3BlYWtlciBudW1iZXIuCiAgICAgIFlvdSBjYW4gYWxzbyBoZWxwIHRoZSBkaWFyaXphdGlvbiBieSBzZXR0aW5nIHRoZSBzcGVha2VycyByYW5nZSB2aWEgdGhlIGBzcGVha2Vyc19hbW91bnRfcmFuZ2VgIHBhcmFtZXRlci4KCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICAgIEEgZGlyZWN0b3J5IG9mIHRoZSBhdWRpbyBmaWxlcywgYSBzaW5nbGUgZmlsZSBvciBhIGxpc3Qgb2YgZmlsZXMgdG8gdHJhbnNjcmliZS4KICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgICAgICAgT25lIG9mIHRoZSBvZmZpY2lhbCBkaWFyaXphdGlvbiBtb2RlbCBuYW1lcyAocmVmZXJyZWQgYXMgZGlhcml6YXRpb24gcGlwZWxpbmVzKSBvZgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgcHlhbm5vdGUuYXVkaW9gIEh1Z2dpbmdmYWNlIHBhZ2UuIERlZmF1bHQ6ICJweWFubm90ZS9zcGVha2VyLWRpYXJpemF0aW9uLTMuMCIuCiAgICA6cGFyYW0gYWNjZXNzX3Rva2VuOiAgICAgICAgIEFuIGFjY2VzcyB0b2tlbiB0byBwYXNzIGZvciB1c2luZyB0aGUgYHB5YW5ub3RlLmF1ZGlvYCBtb2RlbHMuIElmIG5vdCBwcm92aWRlZCwgaXQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSBsb29raW5nIGZvciB0aGUgZW52aXJvbm1lbnQgdmFyaWFibGUgIkhVR0dJTkdfRkFDRV9IVUJfVE9LRU4iLiBJZiBNTFJ1biBpcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhdmFpbGFibGUsIGl0IHdpbGwgbG9vayBmb3IgYSBzZWNyZXQgIkhVR0dJTkdfRkFDRV9IVUJfVE9LRU4iLgogICAgOnBhcmFtIGRldmljZTogICAgICAgICAgICAgICBEZXZpY2UgdG8gbG9hZCB0aGUgbW9kZWwuIENhbiBiZSBvbmUgb2YgeyJjdWRhIiwgImNwdSJ9LiBEZWZhdWx0IHdpbGwgcHJlZmVyICJjdWRhIiBpZgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhdmFpbGFibGUuCiAgICA6cGFyYW0gc3BlYWtlcnNfbGFiZWxzOiAgICAgIExhYmVscyB0byB1c2UgZm9yIHRoZSByZWNvZ25pemVkIHNwZWFrZXJzLiBEZWZhdWx0OiBudW1lcmljIGxhYmVscyAoMCwgMSwgLi4uKS4KICAgIDpwYXJhbSBzZXBhcmF0ZV9ieV9jaGFubmVsczogSWYgZWFjaCBzcGVha2VyIGlzIHNwZWFraW5nIGluIGEgc2VwYXJhdGUgY2hhbm5lbCwgeW91IGNhbiBkaWFyaXplIGVhY2ggY2hhbm5lbCBhbmQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY29tYmluZSB0aGUgcmVzdWx0IGludG8gYSBzaW5nbGUgZGlhcml6YXRpb24uIEVhY2ggbGFiZWwgc2V0IGluIHRoZSBgc3BlYWtlcnNfbGFiZWxzYAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBwYXJhbWV0ZXIgd2lsbCBiZSBhc3NpZ25lZCB0byBhIHNwZWNpZmljIGNoYW5uZWwgYnkgb3JkZXIuCiAgICA6cGFyYW0gc3BlYWtlcl9wcmVmaXg6ICAgICAgIEEgcHJlZml4IHRvIGFkZCBmb3IgdGhlIHNwZWFrZXJzIGxhYmVscy4gVGhpcyBwYXJhbWV0ZXIgaXMgaWdub3JlZCBpZgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgc3BlYWtlcnNfbGFiZWxzYCBpcyBub3QgTm9uZS4gRGVmYXVsdDogInNwZWFrZXIiLgogICAgOnBhcmFtIG1pbmltdW1fc3BlYWtlcnM6ICAgICBTZXQgdGhlIG1pbmltdW0gZXhwZWN0ZWQgYW1vdW50IG9mIHNwZWFrZXJzIHRvIGJlIGluIHRoZSBhdWRpbyBmaWxlcy4gVGhpcyBwYXJhbWV0ZXIgaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaWdub3JlZCBpZiBgc3BlYWtlcnNfbGFiZWxzYCBpcyBub3QgTm9uZS4KICAgIDpwYXJhbSBtYXhpbXVtX3NwZWFrZXJzOiAgICAgU2V0IHRoZSBtYXhpbXVtIGV4cGVjdGVkIGFtb3VudCBvZiBzcGVha2VycyB0byBiZSBpbiB0aGUgYXVkaW8gZmlsZXMuIFRoaXMgcGFyYW1ldGVyIGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGlnbm9yZWQgaWYgYHNwZWFrZXJzX2xhYmVsc2AgaXMgbm90IE5vbmUuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgIFdoZXRoZXIgdG8gcHJlc2VudCBsb2dzIG9mIGEgcHJvZ3Jlc3MgYmFyIGFuZCBlcnJvcnMuIERlZmF1bHQ6IFRydWUuCgogICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CgogICAgICAgICAgICAgICogU3BlZWNoIGRpYXJpemF0aW9uIGRpY3Rpb25hcnkuCiAgICAgICAgICAgICAgKiBBIGRpY3Rpb25hcnkgb2YgZXJyb3JlZCBmaWxlcyB0aGF0IHdlcmUgbm90IHRyYW5zY3JpYmVkLgogICAgIiIiCiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgR2V0IHRoZSBpbnB1dCBhdWRpbyBmaWxlcyB0byBkaWFyaXplOgogICAgaWYgaXNpbnN0YW5jZShkYXRhX3BhdGgsIHN0cik6CiAgICAgICAgZGF0YV9wYXRoID0gcGF0aGxpYi5QYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQogICAgICAgIGF1ZGlvX2ZpbGVzID0gX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQogICAgZWxzZTogICMgU2hvdWxkIGJlIGEgbGlzdCBvZiBmaWxlcy4KICAgICAgICBhdWRpb19maWxlcyA9IGRhdGFfcGF0aAoKICAgICMgR2V0IHRoZSBIdWdnaW5nZmFjZSBhY2Nlc3MgdG9rZW46CiAgICBhY2Nlc3NfdG9rZW4gPSBfZ2V0X2FjY2Vzc190b2tlbihwYXJhbWV0ZXI9YWNjZXNzX3Rva2VuKQogICAgaWYgYWNjZXNzX3Rva2VuIGlzIE5vbmU6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgIkEgSHVnZ2luZ2ZhY2UgYWNjZXNzIHRva2VuIG11c3QgYmUgcHJvdmlkZWQgdG8gdXNlIGBweWFubm90ZS5hdWRpb2AgbW9kZWxzLiBBY2Nlc3MgdG9rZW4gY2FuIGJlIHBhc3NlZCAiCiAgICAgICAgICAgICJ2aWEgb25lIG9mIHRoZSBmb2xsb3dpbmcgb3B0aW9uczpcbiIKICAgICAgICAgICAgIiogVXNlIHRoZSBwYXJhbWV0ZXIgYGFjY2Vzc190b2tlbmAuXG4iCiAgICAgICAgICAgICIqIFNldCBhbiBlbnZpcm9ubWVudCB2YXJpYWJsZSBuYW1lZCAnSFVHR0lOR19GQUNFX0hVQl9UT0tFTicuXG4iCiAgICAgICAgICAgICIqIElmIHVzaW5nIE1MUnVuLCB5b3UgY2FuIHBhc3MgaXQgYXMgYSBzZWNyZXQgbmFtZWQgJ0hVR0dJTkdfRkFDRV9IVUJfVE9LRU4nLiIKICAgICAgICApCgogICAgIyBMb2FkIHRoZSBkaWFyaXphdGlvbiBwaXBlbGluZToKICAgIHBpcGVsaW5lID0gcHlhbm5vdGUuYXVkaW8uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKAogICAgICAgIGNoZWNrcG9pbnRfcGF0aD1tb2RlbF9uYW1lLCB1c2VfYXV0aF90b2tlbj1hY2Nlc3NfdG9rZW4KICAgICkKCiAgICAjIFNldCB0aGUgZGV2aWNlOgogICAgZGV2aWNlID0gZGV2aWNlIG9yICgiY3VkYSIgaWYgdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSBlbHNlICJjcHUiKQogICAgaWYgZGV2aWNlICE9ICJjcHUiOgogICAgICAgIHBpcGVsaW5lLnRvKHRvcmNoLmRldmljZShkZXZpY2UpKQoKICAgICMgUHJlcGFyZSB0aGUgc3VjY2Vzc2VzIGRhdGFmcmFtZSBhbmQgZXJyb3JzIGRpY3Rpb25hcnkgdG8gYmUgcmV0dXJuZWQ6CiAgICBkaWFyaXphdGlvbnMgPSB7fQogICAgZXJyb3JzID0ge30KCiAgICAjIFByZXBhcmUgdGhlIGRpYXJpemF0aW9uIGtleXdvcmQgYXJndW1lbnRzOgogICAgZGlhcml6ZV9rd2FyZ3MgPSB7fQogICAgaWYgc3BlYWtlcnNfbGFiZWxzOgogICAgICAgIGRpYXJpemVfa3dhcmdzWyJudW1fc3BlYWtlcnMiXSA9IGxlbihzcGVha2Vyc19sYWJlbHMpCiAgICBlbHNlOgogICAgICAgIGlmIG1pbmltdW1fc3BlYWtlcnM6CiAgICAgICAgICAgIGRpYXJpemVfa3dhcmdzWyJtaW5fc3BlYWtlcnMiXSA9IG1pbmltdW1fc3BlYWtlcnMKICAgICAgICBpZiBtYXhpbXVtX3NwZWFrZXJzOgogICAgICAgICAgICBkaWFyaXplX2t3YXJnc1sibWF4X3NwZWFrZXJzIl0gPSBtYXhpbXVtX3NwZWFrZXJzCgogICAgIyBHbyBvdmVyIHRoZSBhdWRpbyBmaWxlcyBhbmQgZGlhcml6ZToKICAgIGZvciBhdWRpb19maWxlIGluIHRxZG0oCiAgICAgICAgYXVkaW9fZmlsZXMsIGRlc2M9IkRpYXJpemluZyIsIHVuaXQ9ImZpbGUiLCBkaXNhYmxlPW5vdCB2ZXJib3NlCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgIyBMb2FkIGF1ZGlvIGZpbGU6CiAgICAgICAgICAgIGF1ZGlvLCBzYW1wbGVfcmF0ZSA9IHRvcmNoYXVkaW8ubG9hZCh1cmk9YXVkaW9fZmlsZSwgY2hhbm5lbHNfZmlyc3Q9VHJ1ZSkKICAgICAgICAgICAgIyBHZXQgdGhlIGRpYXJpemF0aW9uIChpZiBwcm92aWRlZCk6CiAgICAgICAgICAgIGRpYXJpemF0aW9uc1thdWRpb19maWxlLm5hbWVdID0gX2RpYXJpemUoCiAgICAgICAgICAgICAgICBhdWRpbz1hdWRpbywKICAgICAgICAgICAgICAgIHNhbXBsZV9yYXRlPXNhbXBsZV9yYXRlLAogICAgICAgICAgICAgICAgcGlwZWxpbmU9cGlwZWxpbmUsCiAgICAgICAgICAgICAgICBzcGVha2Vyc19sYWJlbHM9c3BlYWtlcnNfbGFiZWxzLAogICAgICAgICAgICAgICAgc2VwYXJhdGVfYnlfY2hhbm5lbHM9c2VwYXJhdGVfYnlfY2hhbm5lbHMsCiAgICAgICAgICAgICAgICBzcGVha2VyX3ByZWZpeD1zcGVha2VyX3ByZWZpeCwKICAgICAgICAgICAgICAgIGRpYXJpemVfa3dhcmdzPWRpYXJpemVfa3dhcmdzLAogICAgICAgICAgICApCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBleGNlcHRpb246CiAgICAgICAgICAgICMgTm90ZSB0aGUgZXhjZXB0aW9uIGFzIGVycm9yIGluIHRoZSBkaWN0aW9uYXJ5OgogICAgICAgICAgICBpZiB2ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi53YXJuaW5nKGYiRXJyb3IgaW4gZmlsZTogJ3thdWRpb19maWxlLm5hbWV9JyIpCiAgICAgICAgICAgIGVycm9yc1tzdHIoYXVkaW9fZmlsZS5uYW1lKV0gPSBzdHIoZXhjZXB0aW9uKQogICAgICAgICAgICBjb250aW51ZQoKICAgICMgUHJpbnQgdGhlIGhlYWQgb2YgdGhlIHByb2R1Y2VkIGRhdGFmcmFtZSBhbmQgcmV0dXJuOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJEb25lICh7bGVuKGRpYXJpemF0aW9ucyl9L3tsZW4oYXVkaW9fZmlsZXMpfSlcbiIpCiAgICByZXR1cm4gZGlhcml6YXRpb25zLCBlcnJvcnMKCgpkZWYgX2dldF9hdWRpb19maWxlcygKICAgIGRhdGFfcGF0aDogcGF0aGxpYi5QYXRoLAopIC0+IExpc3RbcGF0aGxpYi5QYXRoXToKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICBhdWRpb19maWxlcyA9IGxpc3QoZGF0YV9wYXRoLmdsb2IoIiouKiIpKQogICAgZWxpZiBkYXRhX3BhdGguaXNfZmlsZSgpOgogICAgICAgIGF1ZGlvX2ZpbGVzID0gW2RhdGFfcGF0aF0KICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJVbnJlY29nbml6ZWQgZGF0YSBwYXRoLiBUaGUgcGFyYW1ldGVyIGBkYXRhX3BhdGhgIG11c3QgYmUgZWl0aGVyIGEgZGlyZWN0b3J5IHBhdGggb3IgYSBmaWxlIHBhdGguICIKICAgICAgICAgICAgZiJHaXZlbjoge3N0cihkYXRhX3BhdGgpfSAiCiAgICAgICAgKQoKICAgIHJldHVybiBhdWRpb19maWxlcwoKCmRlZiBfZ2V0X2FjY2Vzc190b2tlbihwYXJhbWV0ZXI6IHN0cikgLT4gc3RyOgogICAgIyBJZiBnaXZlbiBhcyBhIHBhcmFtZXRlciwgcmV0dXJuIGl0OgogICAgaWYgcGFyYW1ldGVyOgogICAgICAgIHJldHVybiBwYXJhbWV0ZXIKCiAgICAjIE90aGVyd2lzZSwgbG9vayBhdCB0aGUgZW52aXJvbm1lbnQgdmFyaWFibGU6CiAgICBlbnZpcm9ubWVudF92YXJpYWJsZSA9IG9zLmVudmlyb24uZ2V0KCJIVUdHSU5HX0ZBQ0VfSFVCX1RPS0VOIikKICAgIGlmIGVudmlyb25tZW50X3ZhcmlhYmxlOgogICAgICAgIHJldHVybiBlbnZpcm9ubWVudF92YXJpYWJsZQoKICAgICMgTGFzdGx5LCB0cnkgbG9vayBpbiB0aGUgc2V0IHNlY3JldHMgaW4gTUxSdW46CiAgICBzZWNyZXQgPSBOb25lCiAgICB0cnk6CiAgICAgICAgaW1wb3J0IG1scnVuCgogICAgICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJtbHJ1biIpCiAgICAgICAgc2VjcmV0ID0gY29udGV4dC5nZXRfc2VjcmV0KGtleT0iSFVHR0lOR19GQUNFX0hVQl9UT0tFTiIpCiAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvcjoKICAgICAgICBwYXNzCgogICAgcmV0dXJuIHNlY3JldAoKCmRlZiBfZGlhcml6ZSgKICAgIGF1ZGlvOiB0b3JjaC5UZW5zb3IsCiAgICBzYW1wbGVfcmF0ZTogaW50LAogICAgcGlwZWxpbmU6IHB5YW5ub3RlLmF1ZGlvLlBpcGVsaW5lLAogICAgc3BlYWtlcnNfbGFiZWxzOiBMaXN0W3N0cl0sCiAgICBzZXBhcmF0ZV9ieV9jaGFubmVsczogYm9vbCwKICAgIHNwZWFrZXJfcHJlZml4OiBzdHIsCiAgICBkaWFyaXplX2t3YXJnczogZGljdCwKKSAtPiBMaXN0W1R1cGxlW2Zsb2F0LCBmbG9hdCwgc3RyXV06CiAgICAjIElmIHRoZXJlIGlzIG5vIG5lZWQgZm9yIHNlcGFyYXRpb24gYnkgY2hhbm5lbHMsIHdlIGRpYXJpemUgYW5kIHJldHVybjoKICAgIGlmIG5vdCBzZXBhcmF0ZV9ieV9jaGFubmVsczoKICAgICAgICAjIERpYXJpemU6CiAgICAgICAgZGlhcml6YXRpb246IHB5YW5ub3RlLmNvcmUuQW5ub3RhdGlvbiA9IHBpcGVsaW5lKAogICAgICAgICAgICBmaWxlPXsid2F2ZWZvcm0iOiBhdWRpbywgInNhbXBsZV9yYXRlIjogc2FtcGxlX3JhdGV9LCAqKmRpYXJpemVfa3dhcmdzCiAgICAgICAgKQogICAgICAgICMgVmVyaWZ5IHNwZWFrZXJzIGxhYmVscyAoc2hvdWxkIG5vdCBmYWlsIGhlcmUgYXMgd2Ugc2V0IGBudW1fc3BlYWtlcnM9bGVuKHNwZWFrZXJzX2xhYmVscylgIHdoZW4gaW5mZXJyaW5nCiAgICAgICAgIyB0aHJvdWdoIHRoZSBwaXBlbGluZSk6CiAgICAgICAgaWYgc3BlYWtlcnNfbGFiZWxzOgogICAgICAgICAgICBnaXZlbl9zcGVha2VycyA9IGxlbihzcGVha2Vyc19sYWJlbHMpCiAgICAgICAgICAgIGZvdW5kX3NwZWFrZXJzID0gbGVuKHNldChkaWFyaXphdGlvbi5sYWJlbHMoKSkpCiAgICAgICAgICAgIGlmIGdpdmVuX3NwZWFrZXJzIDwgZm91bmRfc3BlYWtlcnM6CiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgICAgIGYiTm90IGVub3VnaCBgc3BlYWtlcnNfbGFiZWxzYCB3ZXJlIGdpdmVuLiBHb3Qge2dpdmVuX3NwZWFrZXJzfSBsYWJlbHMgYnV0IHRoZSBkaWFyaXphdGlvbiAiCiAgICAgICAgICAgICAgICAgICAgZiJyZWNvZ25pemVkIHtmb3VuZF9zcGVha2Vyc30gc3BlYWtlcnMuIgogICAgICAgICAgICAgICAgKQogICAgICAgICMgUmV0dXJuIGFzIGEgZGlhcml6YXRpb24gbGlzdCAtIGEgc29ydGVkIGxpc3Qgb2YgdHVwbGVzIG9mIHN0YXJ0IHRpbWUsIGVuZCB0aW1lIGFuZCBhIGxhYmVsICh0aGUgZGVmYXVsdCBsYWJlbAogICAgICAgICMgcmV0dXJuZWQgaXMgIlNQRUFLRVJfaSIgc28gd2UgdGFrZSBvbmx5IHRoZSBpbmRleCBvdXQgb2YgaXQpOgogICAgICAgIHJldHVybiBbCiAgICAgICAgICAgICgKICAgICAgICAgICAgICAgIHNlZ21lbnQuc3RhcnQsCiAgICAgICAgICAgICAgICBzZWdtZW50LmVuZCwKICAgICAgICAgICAgICAgIHNwZWFrZXJzX2xhYmVsc1tpbnQobGFiZWwuc3BsaXQoIl8iKVsxXSldCiAgICAgICAgICAgICAgICBpZiBzcGVha2Vyc19sYWJlbHMKICAgICAgICAgICAgICAgIGVsc2UgZiJ7c3BlYWtlcl9wcmVmaXh9e2ludChsYWJlbC5zcGxpdCgnXycpWzFdKX0iLAogICAgICAgICAgICApCiAgICAgICAgICAgIGZvciBzZWdtZW50LCB0cmFjaywgbGFiZWwgaW4gZGlhcml6YXRpb24uaXRlcnRyYWNrcyh5aWVsZF9sYWJlbD1UcnVlKQogICAgICAgIF0KCiAgICAjIFNlcGFyYXRlIHRvIGNoYW5uZWxzIGFuZCBkaWFyaXplICh3ZSBleHBlY3Qgb25seSBvbmUgc3BlYWtlciBwZXIgY2hhbm5lbCk6CiAgICBjaGFubmVsX2RpYXJpemF0aW9ucyA9IFsKICAgICAgICBfZGlhcml6ZSgKICAgICAgICAgICAgYXVkaW89YXVkaW9bY2hhbm5lbF0udW5zcXVlZXplKAogICAgICAgICAgICAgICAgMAogICAgICAgICAgICApLCAgIyBUYWtlIGNoYW5uZWwgYW5kIGFkZCBhIGNoYW5uZWwgZGltZW5zaW9uIHRvIGl0LgogICAgICAgICAgICBzYW1wbGVfcmF0ZT1zYW1wbGVfcmF0ZSwKICAgICAgICAgICAgcGlwZWxpbmU9cGlwZWxpbmUsCiAgICAgICAgICAgIHNwZWFrZXJzX2xhYmVscz1bCiAgICAgICAgICAgICAgICBzcGVha2Vyc19sYWJlbHNbY2hhbm5lbF0KICAgICAgICAgICAgXSwgICMgVGFrZSB0aGUgY2hhbm5lbCdzIGxhYmVsIG9ubHkuCiAgICAgICAgICAgIHNlcGFyYXRlX2J5X2NoYW5uZWxzPUZhbHNlLAogICAgICAgICAgICBzcGVha2VyX3ByZWZpeD1zcGVha2VyX3ByZWZpeCwKICAgICAgICAgICAgZGlhcml6ZV9rd2FyZ3M9eyJudW1fc3BlYWtlcnMiOiAxfSwgICMgU2V0IHRvIG9uZSBzcGVha2VyLgogICAgICAgICkKICAgICAgICBmb3IgY2hhbm5lbCBpbiByYW5nZShhdWRpby5zaGFwZVswXSkKICAgIF0KCiAgICAjIE1lcmdlIHRoZSBjaGFubmVsIGRpYXJpemF0aW9ucyBpbnRvIGEgc2luZ2xlIHNvcnRlZCBsaXN0OgogICAgcmV0dXJuIGxpc3QoaGVhcHEubWVyZ2UoKmNoYW5uZWxfZGlhcml6YXRpb25zKSkK - commands: [] - code_origin: '' - origin_filename: '' - requirements: [] - entry_points: - open_mpi_handler: - name: open_mpi_handler - doc: '' - parameters: - - name: worker_inputs - type: List[str] - default: '' - - name: root_worker_inputs - type: Dict[str, Any] - default: null - outputs: - - default: '' - lineno: 59 - decorator: - name: decorator - doc: '' - parameters: - - name: handler - default: '' - outputs: - - default: '' - lineno: 71 - wrapper: - name: wrapper - doc: '' - parameters: [] - outputs: - - default: '' - lineno: 76 - diarize: - name: diarize - doc: "Perform speech diarization on given audio files using pyannote-audio (https://github.com/pyannote/pyannote-audio).\n\ - The end result is a dictionary with the file names as keys and their diarization\ - \ as value. A diarization is a list\nof tuples: (start, end, speaker_label).\n\ - \nTo use the `pyannote.audio` models you must pass a Huggingface token and\ - \ get access to the required models. The\ntoken can be passed in one of the\ - \ following options:\n\n* Use the parameter `access_token`.\n* Set an environment\ - \ variable named \"HUGGING_FACE_HUB_TOKEN\".\n* If using MLRun, you can pass\ - \ it as a secret named \"HUGGING_FACE_HUB_TOKEN\".\n\nTo get access to the\ - \ models on Huggingface, visit their page. For example, to use the default\ - \ diarization model set\nin this function (\"pyannote/speaker-diarization-3.0\"\ - ), you need access for these two models:\n\n* https://huggingface.co/pyannote/segmentation-3.0\n\ - * https://huggingface.co/pyannote/speaker-diarization-3.0\n\nNote: To control\ - \ the recognized speakers in the diarization output you can choose one of\ - \ the following methods:\n\n* For a known speakers amount, you may set speaker\ - \ labels via the `speakers_labels` parameter that will be used in\n the order\ - \ of speaking in the audio (first person speaking be the first label in the\ - \ list). In addition, you can do\n diarization per channel (setting the parameter\ - \ `separate_by_channels` to True). Each label will be assigned to a\n specific\ - \ channel by order (first label to channel 0, second label to channel 1 and\ - \ so on). Notice, this will\n increase runtime.\n* For unknown speakers amount,\ - \ you can set the `speaker_prefix` parameter to add a prefix for each speaker\ - \ number.\n You can also help the diarization by setting the speakers range\ - \ via the `speakers_amount_range` parameter." - parameters: - - name: data_path - type: Union[str, List[str]] - doc: A directory of the audio files, a single file or a list of files to transcribe. - default: '' - - name: model_name - type: str - doc: 'One of the official diarization model names (referred as diarization - pipelines) of `pyannote.audio` Huggingface page. Default: "pyannote/speaker-diarization-3.0".' - default: pyannote/speaker-diarization-3.0 - - name: access_token - type: str - doc: An access token to pass for using the `pyannote.audio` models. If not - provided, it will be looking for the environment variable "HUGGING_FACE_HUB_TOKEN". - If MLRun is available, it will look for a secret "HUGGING_FACE_HUB_TOKEN". - default: null - - name: device - type: str - doc: Device to load the model. Can be one of {"cuda", "cpu"}. Default will - prefer "cuda" if available. - default: null - - name: speakers_labels - type: List[str] - doc: 'Labels to use for the recognized speakers. Default: numeric labels (0, - 1, ...).' - default: null - - name: speaker_prefix - type: str - doc: 'A prefix to add for the speakers labels. This parameter is ignored if - `speakers_labels` is not None. Default: "speaker".' - default: speaker_ - - name: separate_by_channels - type: bool - doc: If each speaker is speaking in a separate channel, you can diarize each - channel and combine the result into a single diarization. Each label set - in the `speakers_labels` parameter will be assigned to a specific channel - by order. - default: false - - name: minimum_speakers - type: int - doc: Set the minimum expected amount of speakers to be in the audio files. - This parameter is ignored if `speakers_labels` is not None. - default: null - - name: maximum_speakers - type: int - doc: Set the maximum expected amount of speakers to be in the audio files. - This parameter is ignored if `speakers_labels` is not None. - default: null - - name: verbose - type: bool - doc: 'Whether to present logs of a progress bar and errors. Default: True.' - default: false - outputs: - - default: '' - doc: 'A tuple of:' - lineno: 137 - description: speech diarization of audio files - default_handler: diarize - disable_auto_mount: false - clone_target_dir: '' - env: [] - priority_class_name: '' - preemption_mode: prevent - affinity: null - tolerations: null - security_context: {} -verbose: false diff --git a/structured_data_generator/function.yaml b/structured_data_generator/function.yaml index f6c1ea5e0..82f48295e 100644 --- a/structured_data_generator/function.yaml +++ b/structured_data_generator/function.yaml @@ -2,7 +2,7 @@ kind: job metadata: name: structured-data-generator tag: '' - hash: 775c1a59adea52f5a1a4d26c96925c88474015f3 + hash: aa811f5c583d081b71d4da97088837546e29c4a1 project: '' labels: author: zeevr @@ -16,7 +16,7 @@ spec: args: [] image: '' build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgYXN0CmltcG9ydCBvcwoKaW1wb3J0IHRxZG0KZnJvbSBsYW5nY2hhaW4uY2hhdF9tb2RlbHMgaW1wb3J0IENoYXRPcGVuQUkKCgpkZWYgX3NldF9vcGVuYWlfc2VjcmV0cygpIC0+IGJvb2w6CiAgICBrZXkgPSAiT1BFTkFJX0FQSV9LRVkiCiAgICBiYXNlID0gIk9QRU5BSV9BUElfQkFTRSIKICAgICMgQ2hlY2sgaWYgdGhlIGtleSBpcyBhbHJlYWR5IGluIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZXM6CiAgICBpZiBrZXkgaW4gb3MuZW52aXJvbiBhbmQgYmFzZSBpbiBvcy5lbnZpcm9uOgogICAgICAgIHJldHVybiBUcnVlCiAgICAjIENoZWNrIGlmIG1scnVuIGlzIGluc3RhbGxlZDoKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgICAgIHJhaXNlIEVudmlyb25tZW50RXJyb3IoCiAgICAgICAgICAgIGYiT25lIG9yIG1vcmUgb2YgdGhlIE9wZW5BSSByZXF1aXJlZCBlbnZpcm9ubWVudCB2YXJpYWJsZXMgKCd7a2V5fScsICd7YmFzZX0nKSBhcmUgbWlzc2luZy4iCiAgICAgICAgICAgIGYiUGxlYXNlIHNldCB0aGVtIGFzIGVudmlyb25tZW50IHZhcmlhYmxlcyBvciBpbnN0YWxsIG1scnVuIChgcGlwIGluc3RhbGwgbWxydW5gKSIKICAgICAgICAgICAgZiJhbmQgc2V0IHRoZW0gYXMgcHJvamVjdCBzZWNyZXRzIHVzaW5nIGBwcm9qZWN5LnNldF9zZWNyZXRzYC4iCiAgICAgICAgKQoKICAgICMgQ2hlY2sgaWYgdGhlIGtleSBpcyBpbiB0aGUgc2VjcmV0czoKICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2NvbnRleHQobmFtZT0iY29udGV4dCIpCiAgICBvcGVuYWlfa2V5ID0gY29udGV4dC5nZXRfc2VjcmV0KGtleSwgTm9uZSkKICAgIG9wZW5haV9iYXNlID0gY29udGV4dC5nZXRfc2VjcmV0KGJhc2UsIE5vbmUpCgogICAgIyBJZiB0aGUga2V5IGlzIG5vdCBpbiB0aGUgc2VjcmV0cywgcmV0dXJuIEZhbHNlOgogICAgaWYgbm90IG9wZW5haV9rZXk6CiAgICAgICAgcmFpc2UgRW52aXJvbm1lbnRFcnJvcigKICAgICAgICAgICAgZiJDb3VsZCBub3QgZmluZCBPcGVuQUkgQVBJIGtleSBpbiB0aGUgZW52aXJvbm1lbnQgdmFyaWFibGVzIG9yIHNlY3JldHMsIgogICAgICAgICAgICBmIiBwbGVhc2Ugc2V0IGl0IGFzOiB7a2V5fS4iCiAgICAgICAgKQogICAgaWYgbm90IG9wZW5haV9iYXNlOgogICAgICAgIHJhaXNlIEVudmlyb25tZW50RXJyb3IoCiAgICAgICAgICAgIGYiQ291bGQgbm90IGZpbmQgT3BlbkFJIEFQSSBiYXNlIGluIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZXMgb3Igc2VjcmV0cywiCiAgICAgICAgICAgIGYiIHBsZWFzZSBzZXQgaXQgYXM6IHtiYXNlfS4iCiAgICAgICAgKQogICAgIyBJZiB0aGUga2V5IGlzIGluIHRoZSBzZWNyZXRzLCBzZXQgaXQgaW4gdGhlIGVudmlyb25tZW50IHZhcmlhYmxlcyBhbmQgcmV0dXJuIFRydWU6CiAgICBvcy5lbnZpcm9uW2tleV0gPSBvcGVuYWlfa2V5CiAgICBvcy5lbnZpcm9uW2Jhc2VdID0gb3BlbmFpX2Jhc2UKICAgIHJldHVybiBUcnVlCgoKZGVmIGdlbmVyYXRlX2RhdGEoCiAgICBmaWVsZHM6IGxpc3QsCiAgICBhbW91bnQ6IGludCA9IDEwLAogICAgbW9kZWxfbmFtZTogc3RyID0gImdwdC0zLjUtdHVyYm8iLAogICAgbGFuZ3VhZ2U6IHN0ciA9ICJlbiIsCiAgICBjaHVua19zaXplOiBpbnQgPSA1MCwKKSAtPiBsaXN0OgogICAgIiIiCiAgICBTdHJ1Y3R1cmVkIGRhdGEgb2YgZWxlbWVudHMgYWNjb3JkaW5nIHRvIHRoZSBnaXZlbiBwYXJhbWV0ZXJzLgogICAgVGhlIGRhdGEgY2FuIGJlIGxhdGVyIGxvZ2dlZCBhcyBhIHN0cnVjdHVyZWQgZmlsZSB3aXRoIE1MUnVuJ3MgYHJldHVybnNgIHBhcmFtZXRlci4KCiAgICA6cGFyYW0gZmllbGRzOiBBIGxpc3Qgb2YgZmllbGRzIHRvIHJhbmRvbWx5IGdlbmVyYXRlLgogICAgOnBhcmFtIGFtb3VudDogVGhlIG51bWJlciBvZiB2YXJpYW50cyB0byBnZW5lcmF0ZS4KICAgIDpwYXJhbSBtb2RlbF9uYW1lOiBUaGUgbmFtZSBvZiB0aGUgbW9kZWwgdG8gdXNlIGZvciBjb252ZXJzYXRpb24gZ2VuZXJhdGlvbi4KICAgICAgICAgICAgICAgICAgICAgICBZb3Ugc2hvdWxkIGNob29zZSBvbmUgb2YgR1BULTQgb3IgR1BULTMuNSBmcm9tIHRoZSBsaXN0IGhlcmU6IGh0dHBzOi8vcGxhdGZvcm0ub3BlbmFpLmNvbS9kb2NzL21vZGVscy4KICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0OiAnZ3B0LTMuNS10dXJibycuCiAgICA6cGFyYW0gbGFuZ3VhZ2U6IFRoZSBsYW5ndWFnZSB0byB1c2UgZm9yIHRoZSBnZW5lcmF0ZWQgY29udmVyc2F0aW9uIHRleHQuCiAgICA6cGFyYW0gY2h1bmtfc2l6ZTogTnVtYmVyIG9mIHNhbXBsZXMgZ2VuZXJhdGVkIGF0IGVhY2ggR1BUIHF1ZXJ5LgogICAgIiIiCiAgICBpbnN0cnVjdGlvbnMgPSAiIgogICAgZm9yIGZpZWxkIGluIGZpZWxkczoKICAgICAgICAjIFNwbGl0IHRoZSBmaWVsZCB0byBrZXkgYW5kIGluc3RydWN0aW9uOgogICAgICAgIGlmICI6IiBpbiBmaWVsZDoKICAgICAgICAgICAga2V5LCBpbnN0cnVjdGlvbiA9IGZpZWxkLnNwbGl0KCI6IiwgMSkKICAgICAgICBlbHNlOgogICAgICAgICAgICBrZXksIGluc3RydWN0aW9uID0gZmllbGQsICJubyBzcGVjaWFsIGluc3RydWN0aW9uIgogICAgICAgICMgUmVwbGFjZSBzcGFjZXMgd2l0aCB1bmRlcnNjb3JlcyBmb3IgdGhlIGtleSB0byBiZSB1c2VkIGFzIGEganNvbiBrZXk6CiAgICAgICAga2V5ID0ga2V5LnJlcGxhY2UoIiAiLCAiXyIpCiAgICAgICAgaW5zdHJ1Y3Rpb25zICs9IGYiKiB7a2V5fToge2luc3RydWN0aW9ufVxuIgoKICAgICMgQ3JlYXRlIHRoZSBwcm9tcHQgc3RydWN0dXJlOgogICAgcHJvbXB0X3N0cnVjdHVyZSA9ICgKICAgICAgICBmImdlbmVyYXRlIHRoZSBmb2xsb3dpbmcgdmFsdWVzIHthbW91bnR9IHRpbWVzIHJhbmRvbWx5LCBpbiBhbiBvcmRlciB0aGF0IGNyZWF0ZXMgYSBqc29uIHRhYmxlLlxuIgogICAgICAgIGYiVXNlIHRoZSBmb2xsb3dpbmcga2V5cyBhbmQgaW5zdHJ1Y3Rpb25zIChleGFtcGxlOiAna2V5OiBpbnN0cnVjdGlvbiBvciBubyBzcGVjaWFsIGluc3RydWN0aW9uJyk6ICIKICAgICAgICBmIntpbnN0cnVjdGlvbnN9LlxuIgogICAgICAgIGYiUGxlYXNlIGdlbmVyYXRlIHRoZSB2YWx1ZXMgaW4ge2xhbmd1YWdlfSBsYW5ndWFnZS4gXG4iCiAgICAgICAgZiJNYWtlIHN1cmUgdGhlIG5hbWVzIG9mIHRoZSBrZXlzIGFyZSB0aGUgc2FtZSBhcyB0aGUgZ2l2ZW4gZmllbGQgbmFtZS5cbiIKICAgICAgICBmIlBsZWFzZSByZXR1cm4gb25seSB0aGUganNvbiBmb3JtYXQgd2l0aG91dCBhbnkgaW50cm9kdWN0aW9uIGFuZCBlbmRpbmciCiAgICApCgogICAgIyBTZXQgdGhlIE9wZW5BSSBzZWNyZXRzOgogICAgX3NldF9vcGVuYWlfc2VjcmV0cygpCgogICAgIyBMb2FkIHRoZSBPcGVuQUkgbW9kZWwgdXNpbmcgbGFuZ2NoYWluOgogICAgbGxtID0gQ2hhdE9wZW5BSShtb2RlbD1tb2RlbF9uYW1lKQoKICAgICMgU3RhcnQgZ2VuZXJhdGluZyBkYXRhOgogICAgZGF0YSA9IFtdCiAgICBmb3IgXyBpbiB0cWRtLnRxZG0ocmFuZ2UoKGFtb3VudCAvLyBjaHVua19zaXplKSArIDEpLCBkZXNjPSJHZW5lcmF0aW5nIik6CiAgICAgICAgIyBXZSB0cnkgdG8gZ2VuZXJhdGUgdGhlIGRhdGEgMyB0aW1lcywgaWYgd2UgZmFpbCB3ZSByYWlzZSBhbiBlcnJvcjoKICAgICAgICBmb3IgdHJ5b3V0IGluIHJhbmdlKDMpOgogICAgICAgICAgICAjIElmIHRoZSBhbW91bnQgd2FudGVkIGlzIGJpZ2dlciB0aGFuIHRoZSBjaHVuayBzaXplLCB3ZSBnZW5lcmF0ZSBhIGNodW5rIG9mIGRhdGEgaW4gdGhlIHNpemUgb2YgdGhlIGNodW5rCiAgICAgICAgICAgICMgYW5kIGRlY3JlYXNlIHRoZSBhbW91bnQgYnkgdGhlIGNodW5rIHNpemUuCiAgICAgICAgICAgICMgb3RoZXJ3aXNlIHdlIGdlbmVyYXRlIGEgY2h1bmsgb2YgZGF0YSBpbiB0aGUgc2l6ZSBvZiB0aGUgYW1vdW50OgogICAgICAgICAgICBpZiBhbW91bnQgPiBjaHVua19zaXplOgogICAgICAgICAgICAgICAgY3VycmVudF9jaHVua19zaXplID0gY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgYW1vdW50IC09IGNodW5rX3NpemUKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGN1cnJlbnRfY2h1bmtfc2l6ZSA9IGFtb3VudAoKICAgICAgICAgICAgIyBDcmVhdGUgdGhlIHByb21wdDoKICAgICAgICAgICAgcHJvbXB0ID0gcHJvbXB0X3N0cnVjdHVyZS5mb3JtYXQoCiAgICAgICAgICAgICAgICBhbW91bnQ9Y3VycmVudF9jaHVua19zaXplLAogICAgICAgICAgICApCgogICAgICAgICAgICAjIEdlbmVyYXRlIGEgY2h1bmsgb2YgZGF0YToKICAgICAgICAgICAgY2h1bmtfZGF0YSA9IGxsbS5wcmVkaWN0KHRleHQ9cHJvbXB0KQoKICAgICAgICAgICAgIyBWYWxpZGF0ZSB0aGUgcmVzcG9uc2UgZm9yIGNvcnJlY3QgcHl0aG9uIGBsaXN0YCBzdHJ1Y3R1cmUKICAgICAgICAgICAgY2h1bmtfZGF0YSA9IGNodW5rX2RhdGFbY2h1bmtfZGF0YS5maW5kKCJbIikgOiBjaHVua19kYXRhLnJmaW5kKCJdIikgKyAxXQogICAgICAgICAgICBpZiBjaHVua19kYXRhLmNvdW50KCJbIikgIT0gY2h1bmtfZGF0YS5jb3VudCgiXSIpOgogICAgICAgICAgICAgICAgcHJpbnQoCiAgICAgICAgICAgICAgICAgICAgIkZhaWxlZCB0byBnZXQgcHJvcGVyIGpzb24gZm9ybWF0IGZyb20gbW9kZWwsIG51bWJlciBvZiAnWycgZG9lc24ndCBtYXRjaCBudW1iZXIgb2YgJ10nLiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgICAgIGNodW5rX2RhdGEgPSBhc3QubGl0ZXJhbF9ldmFsKGNodW5rX2RhdGEpCiAgICAgICAgICAgIGRhdGEgKz0gY2h1bmtfZGF0YQogICAgICAgICAgICBicmVhawogICAgICAgIGlmIHRyeW91dCA9PSAzOgogICAgICAgICAgICByYWlzZSBSdW50aW1lRXJyb3IoCiAgICAgICAgICAgICAgICBmIkNvdWxkIG5vdCBnZW5lcmF0ZSBhIHByb3BlciBqc29uIGZvcm1hdCBmb3IgdGhlIGdpdmVuIGZpZWxkcywgdXNpbmcgZ2l2ZW4gbW9kZWw6IHttb2RlbF9uYW1lfS4iCiAgICAgICAgICAgICAgICBmIiBIaW50OiBHcHQtNCB3b3JrcyBiZXN0IGZvciBtb3N0IHNjZW5hcmlvcy4iCiAgICAgICAgICAgICkKICAgIHJldHVybiBkYXRhCg== + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgYXN0CmltcG9ydCBvcwoKaW1wb3J0IHRxZG0KZnJvbSBsYW5nY2hhaW4uY2hhdF9tb2RlbHMgaW1wb3J0IENoYXRPcGVuQUkKCgpkZWYgX3NldF9vcGVuYWlfc2VjcmV0cygpIC0+IGJvb2w6CiAgICBrZXkgPSAiT1BFTkFJX0FQSV9LRVkiCiAgICBiYXNlID0gIk9QRU5BSV9BUElfQkFTRSIKICAgICMgQ2hlY2sgaWYgdGhlIGtleSBpcyBhbHJlYWR5IGluIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZXM6CiAgICBpZiBrZXkgaW4gb3MuZW52aXJvbiBhbmQgYmFzZSBpbiBvcy5lbnZpcm9uOgogICAgICAgIHJldHVybiBUcnVlCiAgICAjIENoZWNrIGlmIG1scnVuIGlzIGluc3RhbGxlZDoKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgICAgIHJhaXNlIEVudmlyb25tZW50RXJyb3IoCiAgICAgICAgICAgIGYiT25lIG9yIG1vcmUgb2YgdGhlIE9wZW5BSSByZXF1aXJlZCBlbnZpcm9ubWVudCB2YXJpYWJsZXMgKCd7a2V5fScsICd7YmFzZX0nKSBhcmUgbWlzc2luZy4iCiAgICAgICAgICAgIGYiUGxlYXNlIHNldCB0aGVtIGFzIGVudmlyb25tZW50IHZhcmlhYmxlcyBvciBpbnN0YWxsIG1scnVuIChgcGlwIGluc3RhbGwgbWxydW5gKSIKICAgICAgICAgICAgZiJhbmQgc2V0IHRoZW0gYXMgcHJvamVjdCBzZWNyZXRzIHVzaW5nIGBwcm9qZWN5LnNldF9zZWNyZXRzYC4iCiAgICAgICAgKQoKICAgICMgQ2hlY2sgaWYgdGhlIGtleSBpcyBpbiB0aGUgc2VjcmV0czoKICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJjb250ZXh0IikKICAgIG9wZW5haV9rZXkgPSBjb250ZXh0LmdldF9zZWNyZXQoa2V5KQogICAgb3BlbmFpX2Jhc2UgPSBjb250ZXh0LmdldF9zZWNyZXQoYmFzZSkKCiAgICAjIElmIHRoZSBrZXkgaXMgbm90IGluIHRoZSBzZWNyZXRzLCByZXR1cm4gRmFsc2U6CiAgICBpZiBub3Qgb3BlbmFpX2tleToKICAgICAgICByYWlzZSBFbnZpcm9ubWVudEVycm9yKAogICAgICAgICAgICBmIkNvdWxkIG5vdCBmaW5kIE9wZW5BSSBBUEkga2V5IGluIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZXMgb3Igc2VjcmV0cywiCiAgICAgICAgICAgIGYiIHBsZWFzZSBzZXQgaXQgYXM6IHtrZXl9LiIKICAgICAgICApCiAgICBpZiBub3Qgb3BlbmFpX2Jhc2U6CiAgICAgICAgcmFpc2UgRW52aXJvbm1lbnRFcnJvcigKICAgICAgICAgICAgZiJDb3VsZCBub3QgZmluZCBPcGVuQUkgQVBJIGJhc2UgaW4gdGhlIGVudmlyb25tZW50IHZhcmlhYmxlcyBvciBzZWNyZXRzLCIKICAgICAgICAgICAgZiIgcGxlYXNlIHNldCBpdCBhczoge2Jhc2V9LiIKICAgICAgICApCiAgICAjIElmIHRoZSBrZXkgaXMgaW4gdGhlIHNlY3JldHMsIHNldCBpdCBpbiB0aGUgZW52aXJvbm1lbnQgdmFyaWFibGVzIGFuZCByZXR1cm4gVHJ1ZToKICAgIG9zLmVudmlyb25ba2V5XSA9IG9wZW5haV9rZXkKICAgIG9zLmVudmlyb25bYmFzZV0gPSBvcGVuYWlfYmFzZQogICAgcmV0dXJuIFRydWUKCgpkZWYgZ2VuZXJhdGVfZGF0YSgKICAgIGZpZWxkczogbGlzdCwKICAgIGFtb3VudDogaW50ID0gMTAsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSAiZ3B0LTMuNS10dXJibyIsCiAgICBsYW5ndWFnZTogc3RyID0gImVuIiwKICAgIGNodW5rX3NpemU6IGludCA9IDUwLAopIC0+IGxpc3Q6CiAgICAiIiIKICAgIFN0cnVjdHVyZWQgZGF0YSBvZiBlbGVtZW50cyBhY2NvcmRpbmcgdG8gdGhlIGdpdmVuIHBhcmFtZXRlcnMuCiAgICBUaGUgZGF0YSBjYW4gYmUgbGF0ZXIgbG9nZ2VkIGFzIGEgc3RydWN0dXJlZCBmaWxlIHdpdGggTUxSdW4ncyBgcmV0dXJuc2AgcGFyYW1ldGVyLgoKICAgIDpwYXJhbSBmaWVsZHM6IEEgbGlzdCBvZiBmaWVsZHMgdG8gcmFuZG9tbHkgZ2VuZXJhdGUuCiAgICA6cGFyYW0gYW1vdW50OiBUaGUgbnVtYmVyIG9mIHZhcmlhbnRzIHRvIGdlbmVyYXRlLgogICAgOnBhcmFtIG1vZGVsX25hbWU6IFRoZSBuYW1lIG9mIHRoZSBtb2RlbCB0byB1c2UgZm9yIGNvbnZlcnNhdGlvbiBnZW5lcmF0aW9uLgogICAgICAgICAgICAgICAgICAgICAgIFlvdSBzaG91bGQgY2hvb3NlIG9uZSBvZiBHUFQtNCBvciBHUFQtMy41IGZyb20gdGhlIGxpc3QgaGVyZTogaHR0cHM6Ly9wbGF0Zm9ybS5vcGVuYWkuY29tL2RvY3MvbW9kZWxzLgogICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQ6ICdncHQtMy41LXR1cmJvJy4KICAgIDpwYXJhbSBsYW5ndWFnZTogVGhlIGxhbmd1YWdlIHRvIHVzZSBmb3IgdGhlIGdlbmVyYXRlZCBjb252ZXJzYXRpb24gdGV4dC4KICAgIDpwYXJhbSBjaHVua19zaXplOiBOdW1iZXIgb2Ygc2FtcGxlcyBnZW5lcmF0ZWQgYXQgZWFjaCBHUFQgcXVlcnkuCiAgICAiIiIKICAgIGluc3RydWN0aW9ucyA9ICIiCiAgICBmb3IgZmllbGQgaW4gZmllbGRzOgogICAgICAgICMgU3BsaXQgdGhlIGZpZWxkIHRvIGtleSBhbmQgaW5zdHJ1Y3Rpb246CiAgICAgICAgaWYgIjoiIGluIGZpZWxkOgogICAgICAgICAgICBrZXksIGluc3RydWN0aW9uID0gZmllbGQuc3BsaXQoIjoiLCAxKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGtleSwgaW5zdHJ1Y3Rpb24gPSBmaWVsZCwgIm5vIHNwZWNpYWwgaW5zdHJ1Y3Rpb24iCiAgICAgICAgIyBSZXBsYWNlIHNwYWNlcyB3aXRoIHVuZGVyc2NvcmVzIGZvciB0aGUga2V5IHRvIGJlIHVzZWQgYXMgYSBqc29uIGtleToKICAgICAgICBrZXkgPSBrZXkuc3RyaXAoKS5yZXBsYWNlKCIgIiwgIl8iKQogICAgICAgIGluc3RydWN0aW9ucyArPSBmIioge2tleX06IHtpbnN0cnVjdGlvbn1cbiIKCiAgICAjIENyZWF0ZSB0aGUgcHJvbXB0IHN0cnVjdHVyZToKICAgIHByb21wdF9zdHJ1Y3R1cmUgPSAoCiAgICAgICAgZiJnZW5lcmF0ZSB0aGUgZm9sbG93aW5nIHZhbHVlcyB7YW1vdW50fSB0aW1lcyByYW5kb21seSwgaW4gYW4gb3JkZXIgdGhhdCBjcmVhdGVzIGEganNvbiB0YWJsZS5cbiIKICAgICAgICBmIlVzZSB0aGUgZm9sbG93aW5nIGtleXMgYW5kIGluc3RydWN0aW9ucyAoZXhhbXBsZTogJ2tleTogaW5zdHJ1Y3Rpb24gb3Igbm8gc3BlY2lhbCBpbnN0cnVjdGlvbicpOiAiCiAgICAgICAgZiJ7aW5zdHJ1Y3Rpb25zfS5cbiIKICAgICAgICBmIlBsZWFzZSBnZW5lcmF0ZSB0aGUgdmFsdWVzIGluIHtsYW5ndWFnZX0gbGFuZ3VhZ2UuIFxuIgogICAgICAgIGYiTWFrZSBzdXJlIHRoZSBuYW1lcyBvZiB0aGUga2V5cyBhcmUgdGhlIHNhbWUgYXMgdGhlIGdpdmVuIGZpZWxkIG5hbWUuXG4iCiAgICAgICAgZiJQbGVhc2UgcmV0dXJuIG9ubHkgdGhlIGpzb24gZm9ybWF0IHdpdGhvdXQgYW55IGludHJvZHVjdGlvbiBhbmQgZW5kaW5nIgogICAgKQoKICAgICMgU2V0IHRoZSBPcGVuQUkgc2VjcmV0czoKICAgIF9zZXRfb3BlbmFpX3NlY3JldHMoKQoKICAgICMgTG9hZCB0aGUgT3BlbkFJIG1vZGVsIHVzaW5nIGxhbmdjaGFpbjoKICAgIGxsbSA9IENoYXRPcGVuQUkobW9kZWw9bW9kZWxfbmFtZSkKCiAgICAjIFN0YXJ0IGdlbmVyYXRpbmcgZGF0YToKICAgIGRhdGEgPSBbXQogICAgZm9yIF8gaW4gdHFkbS50cWRtKHJhbmdlKChhbW91bnQgLy8gY2h1bmtfc2l6ZSkgKyAxKSwgZGVzYz0iR2VuZXJhdGluZyIpOgogICAgICAgICMgV2UgdHJ5IHRvIGdlbmVyYXRlIHRoZSBkYXRhIDMgdGltZXMsIGlmIHdlIGZhaWwgd2UgcmFpc2UgYW4gZXJyb3I6CiAgICAgICAgZm9yIHRyeW91dCBpbiByYW5nZSgzKToKICAgICAgICAgICAgIyBJZiB0aGUgYW1vdW50IHdhbnRlZCBpcyBiaWdnZXIgdGhhbiB0aGUgY2h1bmsgc2l6ZSwgd2UgZ2VuZXJhdGUgYSBjaHVuayBvZiBkYXRhIGluIHRoZSBzaXplIG9mIHRoZSBjaHVuawogICAgICAgICAgICAjIGFuZCBkZWNyZWFzZSB0aGUgYW1vdW50IGJ5IHRoZSBjaHVuayBzaXplLgogICAgICAgICAgICAjIG90aGVyd2lzZSB3ZSBnZW5lcmF0ZSBhIGNodW5rIG9mIGRhdGEgaW4gdGhlIHNpemUgb2YgdGhlIGFtb3VudDoKICAgICAgICAgICAgaWYgYW1vdW50ID4gY2h1bmtfc2l6ZToKICAgICAgICAgICAgICAgIGN1cnJlbnRfY2h1bmtfc2l6ZSA9IGNodW5rX3NpemUKICAgICAgICAgICAgICAgIGFtb3VudCAtPSBjaHVua19zaXplCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBjdXJyZW50X2NodW5rX3NpemUgPSBhbW91bnQKCiAgICAgICAgICAgICMgQ3JlYXRlIHRoZSBwcm9tcHQ6CiAgICAgICAgICAgIHByb21wdCA9IHByb21wdF9zdHJ1Y3R1cmUuZm9ybWF0KAogICAgICAgICAgICAgICAgYW1vdW50PWN1cnJlbnRfY2h1bmtfc2l6ZSwKICAgICAgICAgICAgKQoKICAgICAgICAgICAgIyBHZW5lcmF0ZSBhIGNodW5rIG9mIGRhdGE6CiAgICAgICAgICAgIGNodW5rX2RhdGEgPSBsbG0ucHJlZGljdCh0ZXh0PXByb21wdCkKCiAgICAgICAgICAgICMgVmFsaWRhdGUgdGhlIHJlc3BvbnNlIGZvciBjb3JyZWN0IHB5dGhvbiBgbGlzdGAgc3RydWN0dXJlCiAgICAgICAgICAgIGNodW5rX2RhdGEgPSBjaHVua19kYXRhW2NodW5rX2RhdGEuZmluZCgiWyIpIDogY2h1bmtfZGF0YS5yZmluZCgiXSIpICsgMV0KICAgICAgICAgICAgaWYgY2h1bmtfZGF0YS5jb3VudCgiWyIpICE9IGNodW5rX2RhdGEuY291bnQoIl0iKToKICAgICAgICAgICAgICAgIHByaW50KAogICAgICAgICAgICAgICAgICAgICJGYWlsZWQgdG8gZ2V0IHByb3BlciBqc29uIGZvcm1hdCBmcm9tIG1vZGVsLCBudW1iZXIgb2YgJ1snIGRvZXNuJ3QgbWF0Y2ggbnVtYmVyIG9mICddJy4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICBjaHVua19kYXRhID0gYXN0LmxpdGVyYWxfZXZhbChjaHVua19kYXRhKQogICAgICAgICAgICBkYXRhICs9IGNodW5rX2RhdGEKICAgICAgICAgICAgYnJlYWsKICAgICAgICBpZiB0cnlvdXQgPT0gMzoKICAgICAgICAgICAgcmFpc2UgUnVudGltZUVycm9yKAogICAgICAgICAgICAgICAgZiJDb3VsZCBub3QgZ2VuZXJhdGUgYSBwcm9wZXIganNvbiBmb3JtYXQgZm9yIHRoZSBnaXZlbiBmaWVsZHMsIHVzaW5nIGdpdmVuIG1vZGVsOiB7bW9kZWxfbmFtZX0uIgogICAgICAgICAgICAgICAgZiIgSGludDogR3B0LTQgd29ya3MgYmVzdCBmb3IgbW9zdCBzY2VuYXJpb3MuIgogICAgICAgICAgICApCiAgICByZXR1cm4gZGF0YQo= base_image: mlrun/mlrun commands: [] code_origin: '' diff --git a/structured_data_generator/item.yaml b/structured_data_generator/item.yaml index b854f0834..8b3644fbd 100755 --- a/structured_data_generator/item.yaml +++ b/structured_data_generator/item.yaml @@ -26,4 +26,4 @@ spec: - langchain - tqdm url: '' -version: 1.1.0 +version: 1.3.0 diff --git a/structured_data_generator/structured_data_generator.py b/structured_data_generator/structured_data_generator.py index 2ace492c5..34fa36d49 100644 --- a/structured_data_generator/structured_data_generator.py +++ b/structured_data_generator/structured_data_generator.py @@ -35,9 +35,9 @@ def _set_openai_secrets() -> bool: ) # Check if the key is in the secrets: - context = mlrun.get_or_create_context(name="context") - openai_key = context.get_secret(key, None) - openai_base = context.get_secret(base, None) + context = mlrun.get_or_create_ctx(name="context") + openai_key = context.get_secret(key) + openai_base = context.get_secret(base) # If the key is not in the secrets, return False: if not openai_key: @@ -83,7 +83,7 @@ def generate_data( else: key, instruction = field, "no special instruction" # Replace spaces with underscores for the key to be used as a json key: - key = key.replace(" ", "_") + key = key.strip().replace(" ", "_") instructions += f"* {key}: {instruction}\n" # Create the prompt structure: diff --git a/text_to_audio_generator/function.yaml b/text_to_audio_generator/function.yaml index 25af4d575..df142d2ef 100644 --- a/text_to_audio_generator/function.yaml +++ b/text_to_audio_generator/function.yaml @@ -2,7 +2,7 @@ kind: job metadata: name: text-to-audio-generator tag: '' - hash: f36d56d620c6a69f414c9cb90e42ec012847a607 + hash: 534e34d316098dcb345860a786ea013102150e67 project: '' labels: author: yonatans @@ -14,7 +14,7 @@ spec: args: [] image: '' build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgbG9nZ2luZwppbXBvcnQgcGF0aGxpYgppbXBvcnQgcmFuZG9tCmZyb20gdHlwaW5nIGltcG9ydCBEaWN0LCBMaXN0LCBPcHRpb25hbCwgVHVwbGUsIFVuaW9uCgppbXBvcnQgYmFyawppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgdG9yY2gKaW1wb3J0IHRvcmNoYXVkaW8KaW1wb3J0IHRxZG0KCiMgR2V0IHRoZSBnbG9iYWwgbG9nZ2VyOgpfTE9HR0VSID0gbG9nZ2luZy5nZXRMb2dnZXIoKQoKCmRlZiBnZW5lcmF0ZV9tdWx0aV9zcGVha2Vyc19hdWRpbygKICAgIGRhdGFfcGF0aDogc3RyLAogICAgb3V0cHV0X2RpcmVjdG9yeTogc3RyLAogICAgc3BlYWtlcnM6IFVuaW9uW0xpc3Rbc3RyXSwgRGljdFtzdHIsIGludF1dLAogICAgYXZhaWxhYmxlX3ZvaWNlczogTGlzdFtzdHJdLAogICAgdXNlX2dwdTogYm9vbCA9IFRydWUsCiAgICB1c2Vfc21hbGxfbW9kZWxzOiBib29sID0gRmFsc2UsCiAgICBvZmZsb2FkX2NwdTogYm9vbCA9IEZhbHNlLAogICAgc2FtcGxlX3JhdGU6IGludCA9IDE2MDAwLAogICAgZmlsZV9mb3JtYXQ6IHN0ciA9ICJ3YXYiLAogICAgdmVyYm9zZTogYm9vbCA9IFRydWUsCiAgICBiaXRzX3Blcl9zYW1wbGU6IE9wdGlvbmFsW2ludF0gPSBOb25lLAopIC0+IFR1cGxlW3N0ciwgcGQuRGF0YUZyYW1lLCBkaWN0XToKICAgICIiIgoKICAgIDpwYXJhbSBkYXRhX3BhdGg6ICAgICAgICAgICBQYXRoIHRvIHRoZSB0ZXh0IGZpbGUgb3IgZGlyZWN0b3J5IGNvbnRhaW5pbmcgdGhlIHRleHQgZmlsZXMgdG8gZ2VuZXJhdGUgYXVkaW8gZnJvbS4KICAgIDpwYXJhbSBvdXRwdXRfZGlyZWN0b3J5OiAgICBQYXRoIHRvIHRoZSBkaXJlY3RvcnkgdG8gc2F2ZSB0aGUgZ2VuZXJhdGVkIGF1ZGlvIGZpbGVzIHRvLgogICAgOnBhcmFtIHNwZWFrZXJzOiAgICAgICAgICAgIExpc3QgLyBEaWN0IG9mIHNwZWFrZXJzIHRvIGdlbmVyYXRlIGF1ZGlvIGZvci4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBhIGxpc3QgaXMgZ2l2ZW4sIHRoZSBzcGVha2VycyB3aWxsIGJlIGFzc2lnbmVkIHRvIGNoYW5uZWxzIGluIHRoZSBvcmRlciBnaXZlbi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBkaWN0aW9uYXJ5LCB0aGUga2V5cyB3aWxsIGJlIHRoZSBzcGVha2VycyBhbmQgdGhlIHZhbHVlcyB3aWxsIGJlIHRoZSBjaGFubmVscy4KICAgIDpwYXJhbSBhdmFpbGFibGVfdm9pY2VzOiAgICBMaXN0IG9mIGF2YWlsYWJsZSB2b2ljZXMgdG8gdXNlIGZvciB0aGUgZ2VuZXJhdGlvbi4KICAgICAgICAgICAgICAgICAgICAgICAgU2VlIGhlcmUgZm9yIHRoZSBhdmFpbGFibGUgdm9pY2VzOgogICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3N1bm8tYWkubm90aW9uLnNpdGUvOGI4ZTg3NDllZDUxNGIwY2JmM2Y2OTkwMTM1NDg2ODM/dj1iYzY3Y2ZmNzg2YjA0YjUwYjNjZWI3NTZmZDA1ZjY4YwogICAgOnBhcmFtIHVzZV9ncHU6ICAgICAgICAgICAgIFdoZXRoZXIgdG8gdXNlIHRoZSBHUFUgZm9yIHRoZSBnZW5lcmF0aW9uLgogICAgOnBhcmFtIHVzZV9zbWFsbF9tb2RlbHM6ICAgIFdoZXRoZXIgdG8gdXNlIHRoZSBzbWFsbCBtb2RlbHMgZm9yIHRoZSBnZW5lcmF0aW9uLgogICAgOnBhcmFtIG9mZmxvYWRfY3B1OiAgICAgICAgIFRPRE86IFdoYXQgZG9lcyB0aGlzIGRvPwogICAgOnBhcmFtIHNhbXBsZV9yYXRlOiAgICAgICAgIFRoZSBzYW1wbGluZyByYXRlIG9mIHRoZSBnZW5lcmF0ZWQgYXVkaW8uCiAgICA6cGFyYW0gZmlsZV9mb3JtYXQ6ICAgICAgICAgVGhlIGZvcm1hdCBvZiB0aGUgZ2VuZXJhdGVkIGF1ZGlvIGZpbGVzLgogICAgOnBhcmFtIHZlcmJvc2U6ICAgICAgICAgICAgIFdoZXRoZXIgdG8gcHJpbnQgdGhlIHByb2dyZXNzIG9mIHRoZSBnZW5lcmF0aW9uLgogICAgOnBhcmFtIGJpdHNfcGVyX3NhbXBsZTogICAgIENoYW5nZXMgdGhlIGJpdCBkZXB0aCBmb3IgdGhlIHN1cHBvcnRlZCBmb3JtYXRzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFN1cHBvcnRlZCBvbmx5IGluICJ3YXYiIG9yICJmbGFjIiBmb3JtYXRzLgoKICAgIDpyZXR1cm5zOiAgICAgICAgICAgICAgICAgICBBIHR1cGxlIG9mOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gVGhlIG91dHB1dCBkaXJlY3RvcnkgcGF0aC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIFRoZSBnZW5lcmF0ZWQgYXVkaW8gZmlsZXMgZGF0YWZyYW1lLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gVGhlIGVycm9ycyBkaWN0aW9uYXJ5LgogICAgIiIiCgogICAgZ2xvYmFsIF9MT0dHRVIKICAgIF9MT0dHRVIgPSBfZ2V0X2xvZ2dlcigpCiAgICAjIEdldCB0aGUgaW5wdXQgdGV4dCBmaWxlcyB0byB0dXJuIHRvIGF1ZGlvOgogICAgZGF0YV9wYXRoID0gcGF0aGxpYi5QYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQogICAgdGV4dF9maWxlcyA9IF9nZXRfdGV4dF9maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQoKICAgICMgTG9hZCB0aGUgYmFyayBtb2RlbHMgYWNjb3JkaW5nIHRvIHRoZSBnaXZlbiBjb25maWd1cmF0aW9uczoKICAgIGJhcmsucHJlbG9hZF9tb2RlbHMoCiAgICAgICAgdGV4dF91c2VfZ3B1PXVzZV9ncHUsCiAgICAgICAgdGV4dF91c2Vfc21hbGw9dXNlX3NtYWxsX21vZGVscywKICAgICAgICBjb2Fyc2VfdXNlX2dwdT11c2VfZ3B1LAogICAgICAgIGNvYXJzZV91c2Vfc21hbGw9dXNlX3NtYWxsX21vZGVscywKICAgICAgICBmaW5lX3VzZV9ncHU9dXNlX2dwdSwKICAgICAgICBmaW5lX3VzZV9zbWFsbD11c2Vfc21hbGxfbW9kZWxzLAogICAgICAgIGNvZGVjX3VzZV9ncHU9dXNlX2dwdSwKICAgICAgICBmb3JjZV9yZWxvYWQ9b2ZmbG9hZF9jcHUsCiAgICApCgogICAgIyBDaGVjayBmb3IgcGVyIGNoYW5uZWwgZ2VuZXJhdGlvbjoKICAgIGlmIGlzaW5zdGFuY2Uoc3BlYWtlcnMsIGRpY3QpOgogICAgICAgIHNwZWFrZXJfcGVyX2NoYW5uZWwgPSBUcnVlCiAgICAgICAgIyBTb3J0IHRoZSBnaXZlbiBzcGVha2VycyBieSBjaGFubmVsczoKICAgICAgICBzcGVha2VycyA9IHsKICAgICAgICAgICAgc3BlYWtlcjogY2hhbm5lbAogICAgICAgICAgICBmb3Igc3BlYWtlciwgY2hhbm5lbCBpbiBzb3J0ZWQoc3BlYWtlcnMuaXRlbXMoKSwga2V5PWxhbWJkYSBpdGVtOiBpdGVtWzFdKQogICAgICAgIH0KICAgIGVsc2U6CiAgICAgICAgc3BlYWtlcl9wZXJfY2hhbm5lbCA9IEZhbHNlCgogICAgIyBQcmVwYXJlIHRoZSByZXNhbXBsaW5nIG1vZHVsZToKICAgIHJlc2FtcGxlciA9IHRvcmNoYXVkaW8udHJhbnNmb3Jtcy5SZXNhbXBsZSgKICAgICAgICBvcmlnX2ZyZXE9YmFyay5TQU1QTEVfUkFURSwgbmV3X2ZyZXE9c2FtcGxlX3JhdGUsIGR0eXBlPXRvcmNoLmZsb2F0MzIKICAgICkKCiAgICAjIFByZXBhcmUgdGhlIGdhcCBiZXR3ZWVuIGVhY2ggc3BlYWtlcjoKICAgIGdhcF9iZXR3ZWVuX3NwZWFrZXJzID0gbnAuemVyb3MoaW50KDAuNSAqIGJhcmsuU0FNUExFX1JBVEUpKQoKICAgICMgUHJlcGFyZSB0aGUgc3VjY2Vzc2VzIGRhdGFmcmFtZSBhbmQgZXJyb3JzIGRpY3Rpb25hcnkgdG8gYmUgcmV0dXJuZWQ6CiAgICBzdWNjZXNzZXMgPSBbXQogICAgZXJyb3JzID0ge30KCiAgICAjIENyZWF0ZSB0aGUgb3V0cHV0IGRpcmVjdG9yeToKICAgIG91dHB1dF9kaXJlY3RvcnkgPSBwYXRobGliLlBhdGgob3V0cHV0X2RpcmVjdG9yeSkKICAgIG91dHB1dF9kaXJlY3RvcnkubWtkaXIoZXhpc3Rfb2s9VHJ1ZSkKCiAgICAjIFN0YXJ0IGdlbmVyYXRpbmcgYXVkaW86CiAgICAjIEdvIG92ZXIgdGhlIGF1ZGlvIGZpbGVzIGFuZCB0cmFuc2NyaWJlOgogICAgZm9yIHRleHRfZmlsZSBpbiB0cWRtLnRxZG0oCiAgICAgICAgdGV4dF9maWxlcywgZGVzYz0iR2VuZXJhdGluZyIsIHVuaXQ9ImZpbGUiLCBkaXNhYmxlPW5vdCB2ZXJib3NlCiAgICApOgoKICAgICAgICB0cnk6CiAgICAgICAgICAgICMgUmFuZG9taXplIHZvaWNlcyBmb3IgZWFjaCBzcGVha2VyOgogICAgICAgICAgICBjaG9zZW5fdm9pY2VzID0ge30KICAgICAgICAgICAgYXZhaWxhYmxlX3ZvaWNlc19jb3B5ID0gYXZhaWxhYmxlX3ZvaWNlcy5jb3B5KCkKICAgICAgICAgICAgZm9yIHNwZWFrZXIgaW4gc3BlYWtlcnM6CiAgICAgICAgICAgICAgICB2b2ljZSA9IHJhbmRvbS5jaG9pY2UoYXZhaWxhYmxlX3ZvaWNlc19jb3B5KQogICAgICAgICAgICAgICAgY2hvc2VuX3ZvaWNlc1tzcGVha2VyXSA9IHZvaWNlCiAgICAgICAgICAgICAgICBhdmFpbGFibGVfdm9pY2VzX2NvcHkucmVtb3ZlKHZvaWNlKQogICAgICAgICAgICAjIFJlYWQgdGV4dDoKICAgICAgICAgICAgd2l0aCBvcGVuKHRleHRfZmlsZSwgInIiKSBhcyBmcDoKICAgICAgICAgICAgICAgIHRleHQgPSBmcC5yZWFkKCkKICAgICAgICAgICAgIyBQcmVwYXJlIGEgaG9sZGVyIGZvciBhbGwgdGhlIGdlbmVyYXRlZCBwaWVjZXMgKGlmIHBlciBjaGFubmVsIGVhY2ggc3BlYWtlciB3aWxsIGhhdmUgaXRzIG93bik6CiAgICAgICAgICAgIGF1ZGlvX3BpZWNlcyA9ICgKICAgICAgICAgICAgICAgIHtzcGVha2VyOiBbXSBmb3Igc3BlYWtlciBpbiBzcGVha2Vyc30KICAgICAgICAgICAgICAgIGlmIHNwZWFrZXJfcGVyX2NoYW5uZWwKICAgICAgICAgICAgICAgIGVsc2UgeyJhbGwiOiBbXX0KICAgICAgICAgICAgKQoKICAgICAgICAgICAgIyBHZW5lcmF0ZSBhdWRpbyBwZXIgbGluZToKICAgICAgICAgICAgZm9yIGxpbmUgaW4gdGV4dC5zcGxpdGxpbmVzKCk6CiAgICAgICAgICAgICAgICAjIFZhbGlkYXRlIGxpbmUgaXMgaW4gY29ycmVjdCBzcGVha2VyIGZvcm1hdDoKCiAgICAgICAgICAgICAgICBpZiAiOiAiIG5vdCBpbiBsaW5lOgogICAgICAgICAgICAgICAgICAgIGlmIHZlcmJvc2U6CiAgICAgICAgICAgICAgICAgICAgICAgIF9MT0dHRVIud2FybmluZyhmIlNraXBwaW5nIGxpbmU6IHtsaW5lfSIpCiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgICMgU3BsaXQgbGluZSB0byBzcGVha2VyIGFuZCBoaXMgd29yZHM6CiAgICAgICAgICAgICAgICBjdXJyZW50X3NwZWFrZXIsIHNlbnRlbmNlcyA9IGxpbmUuc3BsaXQoIjogIiwgMSkKICAgICAgICAgICAgICAgICMgVmFsaWRhdGUgc3BlYWtlciBpcyBrbm93bjoKICAgICAgICAgICAgICAgIGlmIGN1cnJlbnRfc3BlYWtlciBub3QgaW4gc3BlYWtlcnM6CiAgICAgICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAgICAgZiJVbmtub3duIHNwZWFrZXI6IHtjdXJyZW50X3NwZWFrZXJ9LiBHaXZlbiBzcGVha2VycyBhcmU6IHtzcGVha2Vyc30iCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZm9yIHNlbnRlbmNlIGluIF9zcGxpdF9saW5lKGxpbmU9c2VudGVuY2VzKToKICAgICAgICAgICAgICAgICAgICAjIEdlbmVyYXRlIHdvcmRzIGF1ZGlvOgogICAgICAgICAgICAgICAgICAgIGF1ZGlvID0gYmFyay5nZW5lcmF0ZV9hdWRpbygKICAgICAgICAgICAgICAgICAgICAgICAgc2VudGVuY2UsCiAgICAgICAgICAgICAgICAgICAgICAgIGhpc3RvcnlfcHJvbXB0PWNob3Nlbl92b2ljZXNbY3VycmVudF9zcGVha2VyXSwKICAgICAgICAgICAgICAgICAgICAgICAgc2lsZW50PVRydWUsCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGlmIHNwZWFrZXJfcGVyX2NoYW5uZWw6CiAgICAgICAgICAgICAgICAgICAgICAgIHNpbGVuY2UgPSBucC56ZXJvc19saWtlKGF1ZGlvKQogICAgICAgICAgICAgICAgICAgICAgICBmb3Igc3BlYWtlciBpbiBhdWRpb19waWVjZXMua2V5cygpOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgaWYgc3BlYWtlciA9PSBjdXJyZW50X3NwZWFrZXI6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYXVkaW9fcGllY2VzW3NwZWFrZXJdICs9IFthdWRpbywgZ2FwX2JldHdlZW5fc3BlYWtlcnNdCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGF1ZGlvX3BpZWNlc1tzcGVha2VyXSArPSBbc2lsZW5jZSwgZ2FwX2JldHdlZW5fc3BlYWtlcnNdCiAgICAgICAgICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgICAgICAgICAgYXVkaW9fcGllY2VzWyJhbGwiXSArPSBbYXVkaW8sIGdhcF9iZXR3ZWVuX3NwZWFrZXJzXQogICAgICAgICAgICAjIENvbnN0cnVjdCBhIHNpbmdsZSBhdWRpbyBhcnJheSBmcm9tIGFsbCB0aGUgcGllY2VzIGFuZCBjaGFubmVsczoKCiAgICAgICAgICAgIGF1ZGlvID0gbnAudnN0YWNrKAogICAgICAgICAgICAgICAgW25wLmNvbmNhdGVuYXRlKGF1ZGlvX3BpZWNlc1tzcGVha2VyXSkgZm9yIHNwZWFrZXIgaW4gc3BlYWtlcnNdCiAgICAgICAgICAgICkuYXN0eXBlKGR0eXBlPW5wLmZsb2F0MzIpCiAgICAgICAgICAgICMgUmVzYW1wbGU6CiAgICAgICAgICAgIGF1ZGlvID0gdG9yY2guZnJvbV9udW1weShhdWRpbykKICAgICAgICAgICAgYXVkaW8gPSByZXNhbXBsZXIoYXVkaW8pCiAgICAgICAgICAgICMgU2F2ZSB0byBhdWRpbyBmaWxlOgogICAgICAgICAgICBhdWRpb19maWxlID0gb3V0cHV0X2RpcmVjdG9yeSAvIGYie3RleHRfZmlsZS5zdGVtfS57ZmlsZV9mb3JtYXR9IgoKICAgICAgICAgICAgdG9yY2hhdWRpby5zYXZlKAogICAgICAgICAgICAgICAgdXJpPXN0cihhdWRpb19maWxlKSwKICAgICAgICAgICAgICAgIHNyYz1hdWRpbywKICAgICAgICAgICAgICAgIHNhbXBsZV9yYXRlPXNhbXBsZV9yYXRlLAogICAgICAgICAgICAgICAgZm9ybWF0PWZpbGVfZm9ybWF0LAogICAgICAgICAgICAgICAgYml0c19wZXJfc2FtcGxlPWJpdHNfcGVyX3NhbXBsZSwKICAgICAgICAgICAgKQoKICAgICAgICAgICAgIyBDb2xsZWN0IHRvIHRoZSBzdWNjZXNzZXM6CiAgICAgICAgICAgIHN1Y2Nlc3Nlcy5hcHBlbmQoW3RleHRfZmlsZS5uYW1lLCBhdWRpb19maWxlLm5hbWVdKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIE5vdGUgdGhlIGV4Y2VwdGlvbiBhcyBlcnJvciBpbiB0aGUgZGljdGlvbmFyeToKICAgICAgICAgICAgaWYgdmVyYm9zZToKICAgICAgICAgICAgICAgIF9MT0dHRVIud2FybmluZyhmIkVycm9yIGluIGZpbGU6ICd7dGV4dF9maWxlLm5hbWV9JyIpCiAgICAgICAgICAgIHByaW50KGV4Y2VwdGlvbikKICAgICAgICAgICAgZXJyb3JzW3RleHRfZmlsZS5uYW1lXSA9IHN0cihleGNlcHRpb24pCgogICAgIyBDb25zdHJ1Y3QgdGhlIHRyYW5zbGF0aW9ucyBkYXRhZnJhbWU6CiAgICBzdWNjZXNzZXMgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgc3VjY2Vzc2VzLAogICAgICAgIGNvbHVtbnM9WyJ0ZXh0X2ZpbGUiLCAiYXVkaW9fZmlsZSJdLAogICAgKQoKICAgICMgUHJpbnQgdGhlIGhlYWQgb2YgdGhlIHByb2R1Y2VkIGRhdGFmcmFtZSBhbmQgcmV0dXJuOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oCiAgICAgICAgICAgIGYiRG9uZSAoe3N1Y2Nlc3Nlcy5zaGFwZVswXX0ve2xlbih0ZXh0X2ZpbGVzKX0pXG4iCiAgICAgICAgICAgIGYiVHJhbnNsYXRpb25zIHN1bW1hcnk6XG4iCiAgICAgICAgICAgIGYie3N1Y2Nlc3Nlcy5oZWFkKCl9IgogICAgICAgICkKICAgIHJldHVybiBzdHIob3V0cHV0X2RpcmVjdG9yeSksIHN1Y2Nlc3NlcywgZXJyb3JzCgoKZGVmIF9nZXRfdGV4dF9maWxlcygKICAgIGRhdGFfcGF0aDogcGF0aGxpYi5QYXRoLAopIC0+IExpc3RbcGF0aGxpYi5QYXRoXToKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICB0ZXh0X2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgdGV4dF9maWxlcyA9IFtkYXRhX3BhdGhdCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiVW5yZWNvZ25pemVkIGRhdGEgcGF0aC4gVGhlIHBhcmFtZXRlciBgZGF0YV9wYXRoYCBtdXN0IGJlIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgZmlsZSBwYXRoLiAiCiAgICAgICAgICAgIGYiR2l2ZW46IHtzdHIoZGF0YV9wYXRoKX0gIgogICAgICAgICkKCiAgICByZXR1cm4gdGV4dF9maWxlcwoKCmRlZiBfc3BsaXRfbGluZShsaW5lOiBzdHIsIG1heF9sZW5ndGg6IGludCA9IDI1MCkgLT4gTGlzdFtzdHJdOgogICAgaWYgbGVuKGxpbmUpIDwgbWF4X2xlbmd0aDoKICAgICAgICByZXR1cm4gW2xpbmVdCgogICAgc2VudGVuY2VzID0gWwogICAgICAgIGYie3NlbnRlbmNlLnN0cmlwKCl9LiIgZm9yIHNlbnRlbmNlIGluIGxpbmUuc3BsaXQoIi4iKSBpZiBzZW50ZW5jZS5zdHJpcCgpCiAgICBdCgogICAgc3BsaXRzID0gW10KICAgIGN1cnJlbnRfbGVuZ3RoID0gbGVuKHNlbnRlbmNlc1swXSkKICAgIHNwbGl0ID0gc2VudGVuY2VzWzBdCiAgICBmb3Igc2VudGVuY2UgaW4gc2VudGVuY2VzWzE6XToKICAgICAgICBpZiBjdXJyZW50X2xlbmd0aCArIGxlbihzZW50ZW5jZSkgPiBtYXhfbGVuZ3RoOgogICAgICAgICAgICBzcGxpdHMuYXBwZW5kKHNwbGl0KQogICAgICAgICAgICBzcGxpdCA9IHNlbnRlbmNlCiAgICAgICAgICAgIGN1cnJlbnRfbGVuZ3RoID0gbGVuKHNlbnRlbmNlKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGN1cnJlbnRfbGVuZ3RoICs9IGxlbihzZW50ZW5jZSkKICAgICAgICAgICAgc3BsaXQgKz0gIiAiICsgc2VudGVuY2UKICAgIGlmIHNwbGl0OgogICAgICAgIHNwbGl0cy5hcHBlbmQoc3BsaXQpCgogICAgcmV0dXJuIHNwbGl0cwoKCmRlZiBfZ2V0X2xvZ2dlcigpOgogICAgZ2xvYmFsIF9MT0dHRVIKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KICAgICAgICAjIENoZWNrIGlmIE1MUnVuIGlzIGF2YWlsYWJsZToKICAgICAgICBjb250ZXh0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9jdHgobmFtZT0ibWxydW4iKQogICAgICAgIHJldHVybiBjb250ZXh0LmxvZ2dlcgogICAgZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3I6CiAgICAgICAgcmV0dXJuIF9MT0dHRVIK + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgbG9nZ2luZwppbXBvcnQgcGF0aGxpYgppbXBvcnQgcmFuZG9tCmltcG9ydCB0ZW1wZmlsZQpmcm9tIHR5cGluZyBpbXBvcnQgRGljdCwgTGlzdCwgT3B0aW9uYWwsIFR1cGxlLCBVbmlvbgoKaW1wb3J0IGJhcmsKaW1wb3J0IG51bXB5IGFzIG5wCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHRvcmNoCmltcG9ydCB0b3JjaGF1ZGlvCmltcG9ydCB0cWRtCgojIEdldCB0aGUgZ2xvYmFsIGxvZ2dlcjoKX0xPR0dFUiA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCkKCgpkZWYgZ2VuZXJhdGVfbXVsdGlfc3BlYWtlcnNfYXVkaW8oCiAgICBkYXRhX3BhdGg6IHN0ciwKICAgIHNwZWFrZXJzOiBVbmlvbltMaXN0W3N0cl0sIERpY3Rbc3RyLCBpbnRdXSwKICAgIGF2YWlsYWJsZV92b2ljZXM6IExpc3Rbc3RyXSwKICAgIG91dHB1dF9kaXJlY3Rvcnk6IHN0ciA9IE5vbmUsCiAgICB1c2VfZ3B1OiBib29sID0gVHJ1ZSwKICAgIHVzZV9zbWFsbF9tb2RlbHM6IGJvb2wgPSBGYWxzZSwKICAgIG9mZmxvYWRfY3B1OiBib29sID0gRmFsc2UsCiAgICBzYW1wbGVfcmF0ZTogaW50ID0gMTYwMDAsCiAgICBmaWxlX2Zvcm1hdDogc3RyID0gIndhdiIsCiAgICB2ZXJib3NlOiBib29sID0gVHJ1ZSwKICAgIGJpdHNfcGVyX3NhbXBsZTogT3B0aW9uYWxbaW50XSA9IE5vbmUsCikgLT4gVHVwbGVbc3RyLCBwZC5EYXRhRnJhbWUsIGRpY3RdOgogICAgIiIiCiAgICBHZW5lcmF0ZSBhdWRpbyBmaWxlcyBmcm9tIHRleHQgZmlsZXMuCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgIFBhdGggdG8gdGhlIHRleHQgZmlsZSBvciBkaXJlY3RvcnkgY29udGFpbmluZyB0aGUgdGV4dCBmaWxlcyB0byBnZW5lcmF0ZSBhdWRpbyBmcm9tLgogICAgOnBhcmFtIHNwZWFrZXJzOiAgICAgICAgICAgIExpc3QgLyBEaWN0IG9mIHNwZWFrZXJzIHRvIGdlbmVyYXRlIGF1ZGlvIGZvci4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBhIGxpc3QgaXMgZ2l2ZW4sIHRoZSBzcGVha2VycyB3aWxsIGJlIGFzc2lnbmVkIHRvIGNoYW5uZWxzIGluIHRoZSBvcmRlciBnaXZlbi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBkaWN0aW9uYXJ5LCB0aGUga2V5cyB3aWxsIGJlIHRoZSBzcGVha2VycyBhbmQgdGhlIHZhbHVlcyB3aWxsIGJlIHRoZSBjaGFubmVscy4KICAgIDpwYXJhbSBhdmFpbGFibGVfdm9pY2VzOiAgICBMaXN0IG9mIGF2YWlsYWJsZSB2b2ljZXMgdG8gdXNlIGZvciB0aGUgZ2VuZXJhdGlvbi4KICAgICAgICAgICAgICAgICAgICAgICAgU2VlIGhlcmUgZm9yIHRoZSBhdmFpbGFibGUgdm9pY2VzOgogICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3N1bm8tYWkubm90aW9uLnNpdGUvOGI4ZTg3NDllZDUxNGIwY2JmM2Y2OTkwMTM1NDg2ODM/dj1iYzY3Y2ZmNzg2YjA0YjUwYjNjZWI3NTZmZDA1ZjY4YwogICAgOnBhcmFtIG91dHB1dF9kaXJlY3Rvcnk6ICAgIFBhdGggdG8gdGhlIGRpcmVjdG9yeSB0byBzYXZlIHRoZSBnZW5lcmF0ZWQgYXVkaW8gZmlsZXMgdG8uCiAgICA6cGFyYW0gdXNlX2dwdTogICAgICAgICAgICAgV2hldGhlciB0byB1c2UgdGhlIEdQVSBmb3IgdGhlIGdlbmVyYXRpb24uCiAgICA6cGFyYW0gdXNlX3NtYWxsX21vZGVsczogICAgV2hldGhlciB0byB1c2UgdGhlIHNtYWxsIG1vZGVscyBmb3IgdGhlIGdlbmVyYXRpb24uCiAgICA6cGFyYW0gb2ZmbG9hZF9jcHU6ICAgICAgICAgVG8gcmVkdWNlIHRoZSBtZW1vcnkgZm9vdHByaW50LCB0aGUgbW9kZWxzIGNhbiBiZSBvZmZsb2FkZWQgdG8gdGhlIENQVSBhZnRlciBsb2FkaW5nLgogICAgOnBhcmFtIHNhbXBsZV9yYXRlOiAgICAgICAgIFRoZSBzYW1wbGluZyByYXRlIG9mIHRoZSBnZW5lcmF0ZWQgYXVkaW8uCiAgICA6cGFyYW0gZmlsZV9mb3JtYXQ6ICAgICAgICAgVGhlIGZvcm1hdCBvZiB0aGUgZ2VuZXJhdGVkIGF1ZGlvIGZpbGVzLgogICAgOnBhcmFtIHZlcmJvc2U6ICAgICAgICAgICAgIFdoZXRoZXIgdG8gcHJpbnQgdGhlIHByb2dyZXNzIG9mIHRoZSBnZW5lcmF0aW9uLgogICAgOnBhcmFtIGJpdHNfcGVyX3NhbXBsZTogICAgIENoYW5nZXMgdGhlIGJpdCBkZXB0aCBmb3IgdGhlIHN1cHBvcnRlZCBmb3JtYXRzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFN1cHBvcnRlZCBvbmx5IGluICJ3YXYiIG9yICJmbGFjIiBmb3JtYXRzLgoKICAgIDpyZXR1cm5zOiAgICAgICAgICAgICAgICAgICBBIHR1cGxlIG9mOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gVGhlIG91dHB1dCBkaXJlY3RvcnkgcGF0aC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIFRoZSBnZW5lcmF0ZWQgYXVkaW8gZmlsZXMgZGF0YWZyYW1lLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gVGhlIGVycm9ycyBkaWN0aW9uYXJ5LgogICAgIiIiCgogICAgZ2xvYmFsIF9MT0dHRVIKICAgIF9MT0dHRVIgPSBfZ2V0X2xvZ2dlcigpCiAgICAjIEdldCB0aGUgaW5wdXQgdGV4dCBmaWxlcyB0byB0dXJuIHRvIGF1ZGlvOgogICAgZGF0YV9wYXRoID0gcGF0aGxpYi5QYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQogICAgdGV4dF9maWxlcyA9IF9nZXRfdGV4dF9maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQoKICAgICMgTG9hZCB0aGUgYmFyayBtb2RlbHMgYWNjb3JkaW5nIHRvIHRoZSBnaXZlbiBjb25maWd1cmF0aW9uczoKICAgIGJhcmsucHJlbG9hZF9tb2RlbHMoCiAgICAgICAgdGV4dF91c2VfZ3B1PXVzZV9ncHUsCiAgICAgICAgdGV4dF91c2Vfc21hbGw9dXNlX3NtYWxsX21vZGVscywKICAgICAgICBjb2Fyc2VfdXNlX2dwdT11c2VfZ3B1LAogICAgICAgIGNvYXJzZV91c2Vfc21hbGw9dXNlX3NtYWxsX21vZGVscywKICAgICAgICBmaW5lX3VzZV9ncHU9dXNlX2dwdSwKICAgICAgICBmaW5lX3VzZV9zbWFsbD11c2Vfc21hbGxfbW9kZWxzLAogICAgICAgIGNvZGVjX3VzZV9ncHU9dXNlX2dwdSwKICAgICAgICBmb3JjZV9yZWxvYWQ9b2ZmbG9hZF9jcHUsCiAgICApCgogICAgIyBDaGVjayBmb3IgcGVyIGNoYW5uZWwgZ2VuZXJhdGlvbjoKICAgIGlmIGlzaW5zdGFuY2Uoc3BlYWtlcnMsIGRpY3QpOgogICAgICAgIHNwZWFrZXJfcGVyX2NoYW5uZWwgPSBUcnVlCiAgICAgICAgIyBTb3J0IHRoZSBnaXZlbiBzcGVha2VycyBieSBjaGFubmVsczoKICAgICAgICBzcGVha2VycyA9IHsKICAgICAgICAgICAgc3BlYWtlcjogY2hhbm5lbAogICAgICAgICAgICBmb3Igc3BlYWtlciwgY2hhbm5lbCBpbiBzb3J0ZWQoc3BlYWtlcnMuaXRlbXMoKSwga2V5PWxhbWJkYSBpdGVtOiBpdGVtWzFdKQogICAgICAgIH0KICAgIGVsc2U6CiAgICAgICAgc3BlYWtlcl9wZXJfY2hhbm5lbCA9IEZhbHNlCgogICAgIyBQcmVwYXJlIHRoZSByZXNhbXBsaW5nIG1vZHVsZToKICAgIHJlc2FtcGxlciA9IHRvcmNoYXVkaW8udHJhbnNmb3Jtcy5SZXNhbXBsZSgKICAgICAgICBvcmlnX2ZyZXE9YmFyay5TQU1QTEVfUkFURSwgbmV3X2ZyZXE9c2FtcGxlX3JhdGUsIGR0eXBlPXRvcmNoLmZsb2F0MzIKICAgICkKCiAgICAjIFByZXBhcmUgdGhlIGdhcCBiZXR3ZWVuIGVhY2ggc3BlYWtlcjoKICAgIGdhcF9iZXR3ZWVuX3NwZWFrZXJzID0gbnAuemVyb3MoaW50KDAuNSAqIGJhcmsuU0FNUExFX1JBVEUpKQoKICAgICMgUHJlcGFyZSB0aGUgc3VjY2Vzc2VzIGRhdGFmcmFtZSBhbmQgZXJyb3JzIGRpY3Rpb25hcnkgdG8gYmUgcmV0dXJuZWQ6CiAgICBzdWNjZXNzZXMgPSBbXQogICAgZXJyb3JzID0ge30KCiAgICAjIENyZWF0ZSB0aGUgb3V0cHV0IGRpcmVjdG9yeToKICAgIGlmIG91dHB1dF9kaXJlY3RvcnkgaXMgTm9uZToKICAgICAgICBvdXRwdXRfZGlyZWN0b3J5ID0gdGVtcGZpbGUubWtkdGVtcCgpCiAgICBvdXRwdXRfZGlyZWN0b3J5ID0gcGF0aGxpYi5QYXRoKG91dHB1dF9kaXJlY3RvcnkpCiAgICBpZiBub3Qgb3V0cHV0X2RpcmVjdG9yeS5leGlzdHMoKToKICAgICAgICBvdXRwdXRfZGlyZWN0b3J5Lm1rZGlyKGV4aXN0X29rPVRydWUsIHBhcmVudHM9VHJ1ZSkKCiAgICAjIFN0YXJ0IGdlbmVyYXRpbmcgYXVkaW86CiAgICAjIEdvIG92ZXIgdGhlIGF1ZGlvIGZpbGVzIGFuZCB0cmFuc2NyaWJlOgogICAgZm9yIHRleHRfZmlsZSBpbiB0cWRtLnRxZG0oCiAgICAgICAgdGV4dF9maWxlcywgZGVzYz0iR2VuZXJhdGluZyIsIHVuaXQ9ImZpbGUiLCBkaXNhYmxlPW5vdCB2ZXJib3NlCiAgICApOgoKICAgICAgICB0cnk6CiAgICAgICAgICAgICMgUmFuZG9taXplIHZvaWNlcyBmb3IgZWFjaCBzcGVha2VyOgogICAgICAgICAgICBjaG9zZW5fdm9pY2VzID0ge30KICAgICAgICAgICAgYXZhaWxhYmxlX3ZvaWNlc19jb3B5ID0gYXZhaWxhYmxlX3ZvaWNlcy5jb3B5KCkKICAgICAgICAgICAgZm9yIHNwZWFrZXIgaW4gc3BlYWtlcnM6CiAgICAgICAgICAgICAgICB2b2ljZSA9IHJhbmRvbS5jaG9pY2UoYXZhaWxhYmxlX3ZvaWNlc19jb3B5KQogICAgICAgICAgICAgICAgY2hvc2VuX3ZvaWNlc1tzcGVha2VyXSA9IHZvaWNlCiAgICAgICAgICAgICAgICBhdmFpbGFibGVfdm9pY2VzX2NvcHkucmVtb3ZlKHZvaWNlKQogICAgICAgICAgICAjIFJlYWQgdGV4dDoKICAgICAgICAgICAgd2l0aCBvcGVuKHRleHRfZmlsZSwgInIiKSBhcyBmcDoKICAgICAgICAgICAgICAgIHRleHQgPSBmcC5yZWFkKCkKICAgICAgICAgICAgIyBQcmVwYXJlIGEgaG9sZGVyIGZvciBhbGwgdGhlIGdlbmVyYXRlZCBwaWVjZXMgKGlmIHBlciBjaGFubmVsIGVhY2ggc3BlYWtlciB3aWxsIGhhdmUgaXRzIG93bik6CiAgICAgICAgICAgIGF1ZGlvX3BpZWNlcyA9ICgKICAgICAgICAgICAgICAgIHtzcGVha2VyOiBbXSBmb3Igc3BlYWtlciBpbiBzcGVha2Vyc30KICAgICAgICAgICAgICAgIGlmIHNwZWFrZXJfcGVyX2NoYW5uZWwKICAgICAgICAgICAgICAgIGVsc2UgeyJhbGwiOiBbXX0KICAgICAgICAgICAgKQoKICAgICAgICAgICAgIyBHZW5lcmF0ZSBhdWRpbyBwZXIgbGluZToKICAgICAgICAgICAgZm9yIGxpbmUgaW4gdGV4dC5zcGxpdGxpbmVzKCk6CiAgICAgICAgICAgICAgICAjIFZhbGlkYXRlIGxpbmUgaXMgaW4gY29ycmVjdCBzcGVha2VyIGZvcm1hdDoKCiAgICAgICAgICAgICAgICBpZiAiOiAiIG5vdCBpbiBsaW5lOgogICAgICAgICAgICAgICAgICAgIGlmIHZlcmJvc2U6CiAgICAgICAgICAgICAgICAgICAgICAgIF9MT0dHRVIud2FybmluZyhmIlNraXBwaW5nIGxpbmU6IHtsaW5lfSIpCiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgICMgU3BsaXQgbGluZSB0byBzcGVha2VyIGFuZCBoaXMgd29yZHM6CiAgICAgICAgICAgICAgICBjdXJyZW50X3NwZWFrZXIsIHNlbnRlbmNlcyA9IGxpbmUuc3BsaXQoIjogIiwgMSkKICAgICAgICAgICAgICAgICMgVmFsaWRhdGUgc3BlYWtlciBpcyBrbm93bjoKICAgICAgICAgICAgICAgIGlmIGN1cnJlbnRfc3BlYWtlciBub3QgaW4gc3BlYWtlcnM6CiAgICAgICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAgICAgZiJVbmtub3duIHNwZWFrZXI6IHtjdXJyZW50X3NwZWFrZXJ9LiBHaXZlbiBzcGVha2VycyBhcmU6IHtzcGVha2Vyc30iCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZm9yIHNlbnRlbmNlIGluIF9zcGxpdF9saW5lKGxpbmU9c2VudGVuY2VzKToKICAgICAgICAgICAgICAgICAgICAjIEdlbmVyYXRlIHdvcmRzIGF1ZGlvOgogICAgICAgICAgICAgICAgICAgIGF1ZGlvID0gYmFyay5nZW5lcmF0ZV9hdWRpbygKICAgICAgICAgICAgICAgICAgICAgICAgc2VudGVuY2UsCiAgICAgICAgICAgICAgICAgICAgICAgIGhpc3RvcnlfcHJvbXB0PWNob3Nlbl92b2ljZXNbY3VycmVudF9zcGVha2VyXSwKICAgICAgICAgICAgICAgICAgICAgICAgc2lsZW50PVRydWUsCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGlmIHNwZWFrZXJfcGVyX2NoYW5uZWw6CiAgICAgICAgICAgICAgICAgICAgICAgIHNpbGVuY2UgPSBucC56ZXJvc19saWtlKGF1ZGlvKQogICAgICAgICAgICAgICAgICAgICAgICBmb3Igc3BlYWtlciBpbiBhdWRpb19waWVjZXMua2V5cygpOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgaWYgc3BlYWtlciA9PSBjdXJyZW50X3NwZWFrZXI6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYXVkaW9fcGllY2VzW3NwZWFrZXJdICs9IFthdWRpbywgZ2FwX2JldHdlZW5fc3BlYWtlcnNdCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGF1ZGlvX3BpZWNlc1tzcGVha2VyXSArPSBbc2lsZW5jZSwgZ2FwX2JldHdlZW5fc3BlYWtlcnNdCiAgICAgICAgICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgICAgICAgICAgYXVkaW9fcGllY2VzWyJhbGwiXSArPSBbYXVkaW8sIGdhcF9iZXR3ZWVuX3NwZWFrZXJzXQogICAgICAgICAgICAjIENvbnN0cnVjdCBhIHNpbmdsZSBhdWRpbyBhcnJheSBmcm9tIGFsbCB0aGUgcGllY2VzIGFuZCBjaGFubmVsczoKCiAgICAgICAgICAgIGF1ZGlvID0gbnAudnN0YWNrKAogICAgICAgICAgICAgICAgW25wLmNvbmNhdGVuYXRlKGF1ZGlvX3BpZWNlc1tzcGVha2VyXSkgZm9yIHNwZWFrZXIgaW4gc3BlYWtlcnNdCiAgICAgICAgICAgICkuYXN0eXBlKGR0eXBlPW5wLmZsb2F0MzIpCiAgICAgICAgICAgICMgUmVzYW1wbGU6CiAgICAgICAgICAgIGF1ZGlvID0gdG9yY2guZnJvbV9udW1weShhdWRpbykKICAgICAgICAgICAgYXVkaW8gPSByZXNhbXBsZXIoYXVkaW8pCiAgICAgICAgICAgICMgU2F2ZSB0byBhdWRpbyBmaWxlOgogICAgICAgICAgICBhdWRpb19maWxlID0gb3V0cHV0X2RpcmVjdG9yeSAvIGYie3RleHRfZmlsZS5zdGVtfS57ZmlsZV9mb3JtYXR9IgoKICAgICAgICAgICAgdG9yY2hhdWRpby5zYXZlKAogICAgICAgICAgICAgICAgdXJpPXN0cihhdWRpb19maWxlKSwKICAgICAgICAgICAgICAgIHNyYz1hdWRpbywKICAgICAgICAgICAgICAgIHNhbXBsZV9yYXRlPXNhbXBsZV9yYXRlLAogICAgICAgICAgICAgICAgZm9ybWF0PWZpbGVfZm9ybWF0LAogICAgICAgICAgICAgICAgYml0c19wZXJfc2FtcGxlPWJpdHNfcGVyX3NhbXBsZSwKICAgICAgICAgICAgKQoKICAgICAgICAgICAgIyBDb2xsZWN0IHRvIHRoZSBzdWNjZXNzZXM6CiAgICAgICAgICAgIHN1Y2Nlc3Nlcy5hcHBlbmQoW3RleHRfZmlsZS5uYW1lLCBhdWRpb19maWxlLm5hbWVdKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIE5vdGUgdGhlIGV4Y2VwdGlvbiBhcyBlcnJvciBpbiB0aGUgZGljdGlvbmFyeToKICAgICAgICAgICAgaWYgdmVyYm9zZToKICAgICAgICAgICAgICAgIF9MT0dHRVIud2FybmluZyhmIkVycm9yIGluIGZpbGU6ICd7dGV4dF9maWxlLm5hbWV9JyIpCiAgICAgICAgICAgIHByaW50KGV4Y2VwdGlvbikKICAgICAgICAgICAgZXJyb3JzW3RleHRfZmlsZS5uYW1lXSA9IHN0cihleGNlcHRpb24pCgogICAgIyBDb25zdHJ1Y3QgdGhlIHRyYW5zbGF0aW9ucyBkYXRhZnJhbWU6CiAgICBzdWNjZXNzZXMgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgc3VjY2Vzc2VzLAogICAgICAgIGNvbHVtbnM9WyJ0ZXh0X2ZpbGUiLCAiYXVkaW9fZmlsZSJdLAogICAgKQoKICAgICMgUHJpbnQgdGhlIGhlYWQgb2YgdGhlIHByb2R1Y2VkIGRhdGFmcmFtZSBhbmQgcmV0dXJuOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oCiAgICAgICAgICAgIGYiRG9uZSAoe3N1Y2Nlc3Nlcy5zaGFwZVswXX0ve2xlbih0ZXh0X2ZpbGVzKX0pXG4iCiAgICAgICAgICAgIGYiVHJhbnNsYXRpb25zIHN1bW1hcnk6XG4iCiAgICAgICAgICAgIGYie3N1Y2Nlc3Nlcy5oZWFkKCl9IgogICAgICAgICkKICAgIHJldHVybiBzdHIob3V0cHV0X2RpcmVjdG9yeSksIHN1Y2Nlc3NlcywgZXJyb3JzCgoKZGVmIF9nZXRfdGV4dF9maWxlcygKICAgIGRhdGFfcGF0aDogcGF0aGxpYi5QYXRoLAopIC0+IExpc3RbcGF0aGxpYi5QYXRoXToKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICB0ZXh0X2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgdGV4dF9maWxlcyA9IFtkYXRhX3BhdGhdCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiVW5yZWNvZ25pemVkIGRhdGEgcGF0aC4gVGhlIHBhcmFtZXRlciBgZGF0YV9wYXRoYCBtdXN0IGJlIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgZmlsZSBwYXRoLiAiCiAgICAgICAgICAgIGYiR2l2ZW46IHtzdHIoZGF0YV9wYXRoKX0gIgogICAgICAgICkKCiAgICByZXR1cm4gdGV4dF9maWxlcwoKCmRlZiBfc3BsaXRfbGluZShsaW5lOiBzdHIsIG1heF9sZW5ndGg6IGludCA9IDI1MCkgLT4gTGlzdFtzdHJdOgogICAgaWYgbGVuKGxpbmUpIDwgbWF4X2xlbmd0aDoKICAgICAgICByZXR1cm4gW2xpbmVdCgogICAgc2VudGVuY2VzID0gWwogICAgICAgIGYie3NlbnRlbmNlLnN0cmlwKCl9LiIgZm9yIHNlbnRlbmNlIGluIGxpbmUuc3BsaXQoIi4iKSBpZiBzZW50ZW5jZS5zdHJpcCgpCiAgICBdCgogICAgc3BsaXRzID0gW10KICAgIGN1cnJlbnRfbGVuZ3RoID0gbGVuKHNlbnRlbmNlc1swXSkKICAgIHNwbGl0ID0gc2VudGVuY2VzWzBdCiAgICBmb3Igc2VudGVuY2UgaW4gc2VudGVuY2VzWzE6XToKICAgICAgICBpZiBjdXJyZW50X2xlbmd0aCArIGxlbihzZW50ZW5jZSkgPiBtYXhfbGVuZ3RoOgogICAgICAgICAgICBzcGxpdHMuYXBwZW5kKHNwbGl0KQogICAgICAgICAgICBzcGxpdCA9IHNlbnRlbmNlCiAgICAgICAgICAgIGN1cnJlbnRfbGVuZ3RoID0gbGVuKHNlbnRlbmNlKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGN1cnJlbnRfbGVuZ3RoICs9IGxlbihzZW50ZW5jZSkKICAgICAgICAgICAgc3BsaXQgKz0gIiAiICsgc2VudGVuY2UKICAgIGlmIHNwbGl0OgogICAgICAgIHNwbGl0cy5hcHBlbmQoc3BsaXQpCgogICAgcmV0dXJuIHNwbGl0cwoKCmRlZiBfZ2V0X2xvZ2dlcigpOgogICAgZ2xvYmFsIF9MT0dHRVIKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KICAgICAgICAjIENoZWNrIGlmIE1MUnVuIGlzIGF2YWlsYWJsZToKICAgICAgICBjb250ZXh0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9jdHgobmFtZT0ibWxydW4iKQogICAgICAgIHJldHVybiBjb250ZXh0LmxvZ2dlcgogICAgZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3I6CiAgICAgICAgcmV0dXJuIF9MT0dHRVIK base_image: mlrun/mlrun commands: [] code_origin: '' @@ -25,15 +25,12 @@ spec: entry_points: generate_multi_speakers_audio: name: generate_multi_speakers_audio - doc: '' + doc: Generate audio files from text files. parameters: - name: data_path type: str doc: Path to the text file or directory containing the text files to generate audio from. - - name: output_directory - type: str - doc: Path to the directory to save the generated audio files to. - name: speakers type: Union[List[str], Dict[str, int]] doc: List / Dict of speakers to generate audio for. If a list is given, the @@ -43,6 +40,10 @@ spec: type: List[str] doc: 'List of available voices to use for the generation. See here for the available voices: https://suno-ai.notion.site/8b8e8749ed514b0cbf3f699013548683?v=bc67cff786b04b50b3ceb756fd05f68c' + - name: output_directory + type: str + doc: Path to the directory to save the generated audio files to. + default: null - name: use_gpu type: bool doc: Whether to use the GPU for the generation. @@ -53,7 +54,8 @@ spec: default: false - name: offload_cpu type: bool - doc: 'TODO: What does this do?' + doc: To reduce the memory footprint, the models can be offloaded to the CPU + after loading. default: false - name: sample_rate type: int @@ -75,8 +77,10 @@ spec: outputs: - doc: 'A tuple of: - The output directory path. - The generated audio files dataframe. - The errors dictionary.' - default: '' - lineno: 30 + type: Tuple[str, pd.DataFrame, dict] + lineno: 31 + has_varargs: false + has_kwargs: false description: Generate audio file from text using different speakers default_handler: generate_multi_speakers_audio disable_auto_mount: false diff --git a/text_to_audio_generator/item.yaml b/text_to_audio_generator/item.yaml index dba7f1e0c..4784a80d2 100644 --- a/text_to_audio_generator/item.yaml +++ b/text_to_audio_generator/item.yaml @@ -24,5 +24,5 @@ spec: - bark - torchaudio url: '' -version: 1.0.0 +version: 1.1.0 test_valid: True diff --git a/text_to_audio_generator/text_to_audio_generator.py b/text_to_audio_generator/text_to_audio_generator.py index ad0e114e8..7602745ee 100644 --- a/text_to_audio_generator/text_to_audio_generator.py +++ b/text_to_audio_generator/text_to_audio_generator.py @@ -14,6 +14,7 @@ import logging import pathlib import random +import tempfile from typing import Dict, List, Optional, Tuple, Union import bark @@ -29,9 +30,9 @@ def generate_multi_speakers_audio( data_path: str, - output_directory: str, speakers: Union[List[str], Dict[str, int]], available_voices: List[str], + output_directory: str = None, use_gpu: bool = True, use_small_models: bool = False, offload_cpu: bool = False, @@ -44,13 +45,13 @@ def generate_multi_speakers_audio( Generate audio files from text files. :param data_path: Path to the text file or directory containing the text files to generate audio from. - :param output_directory: Path to the directory to save the generated audio files to. :param speakers: List / Dict of speakers to generate audio for. If a list is given, the speakers will be assigned to channels in the order given. If dictionary, the keys will be the speakers and the values will be the channels. :param available_voices: List of available voices to use for the generation. See here for the available voices: https://suno-ai.notion.site/8b8e8749ed514b0cbf3f699013548683?v=bc67cff786b04b50b3ceb756fd05f68c + :param output_directory: Path to the directory to save the generated audio files to. :param use_gpu: Whether to use the GPU for the generation. :param use_small_models: Whether to use the small models for the generation. :param offload_cpu: To reduce the memory footprint, the models can be offloaded to the CPU after loading. @@ -108,8 +109,11 @@ def generate_multi_speakers_audio( errors = {} # Create the output directory: + if output_directory is None: + output_directory = tempfile.mkdtemp() output_directory = pathlib.Path(output_directory) - output_directory.mkdir(exist_ok=True) + if not output_directory.exists(): + output_directory.mkdir(exist_ok=True, parents=True) # Start generating audio: # Go over the audio files and transcribe: diff --git a/transcribe/function.yaml b/transcribe/function.yaml index 471dd6f26..40dd2f0e6 100644 --- a/transcribe/function.yaml +++ b/transcribe/function.yaml @@ -2,7 +2,7 @@ kind: job metadata: name: transcribe tag: '' - hash: e7f85ec6e204a54069b4e264003cf59d0cb27bfe + hash: 5cd620de67a936ee8a87cfc1f0b97e19730d0a69 project: '' labels: author: yonatans @@ -14,124 +14,287 @@ spec: args: [] image: '' build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG9wZXJhdG9yCmltcG9ydCBwYXRobGliCmZyb20gZnVuY3Rvb2xzIGltcG9ydCByZWR1Y2UsIHdyYXBzCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIExpdGVyYWwsIE5hbWVkVHVwbGUsIFR1cGxlLCBVbmlvbgoKaW1wb3J0IGZhc3Rlcl93aGlzcGVyCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSB0cWRtIGltcG9ydCB0cWRtCgojIEdldCB0aGUgZ2xvYmFsIGxvZ2dlcjoKX0xPR0dFUiA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCkKCgpkZWYgb3Blbl9tcGlfaGFuZGxlcigKICAgIHdvcmtlcl9pbnB1dHM6IExpc3Rbc3RyXSwgcm9vdF93b3JrZXJfaW5wdXRzOiBEaWN0W3N0ciwgQW55XSA9IE5vbmUKKToKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBDaGVjayBmb3IgTUxSdW4gYW5kIE9wZW5NUEkgYXZhaWxhYmlsaXR5OgogICAgY29udGV4dCwgY29tbSA9IF9jaGVja19tbHJ1bl9hbmRfb3Blbl9tcGkoKQoKICAgICMgQ2hlY2sgaWYgTUxSdW4gaXMgYXZhaWxhYmxlLCBzZXQgdGhlIGdsb2JhbCBsb2dnZXIgdG8gTUxSdW4nczoKICAgIGlmIGNvbnRleHQ6CiAgICAgICAgX0xPR0dFUiA9IGNvbnRleHQubG9nZ2VyCgogICAgZGVmIGRlY29yYXRvcihoYW5kbGVyKToKICAgICAgICBpZiBjb21tIGlzIE5vbmUgb3IgY29tbS5HZXRfc2l6ZSgpID09IDE6CiAgICAgICAgICAgIHJldHVybiBoYW5kbGVyCgogICAgICAgIEB3cmFwcyhoYW5kbGVyKQogICAgICAgIGRlZiB3cmFwcGVyKCoqa3dhcmdzKToKICAgICAgICAgICAgIyBHZXQgdGhlIG9wZW4gbXBpIGVudmlyb25tZW50IHByb3BlcnRpZXM6CiAgICAgICAgICAgIHNpemUgPSBjb21tLkdldF9zaXplKCkKICAgICAgICAgICAgcmFuayA9IGNvbW0uR2V0X3JhbmsoKQoKICAgICAgICAgICAgIyBHaXZlIHRoZSBjb3JyZWN0IGNodW5rIG9mIHRoZSB3b3JrZXJzIGlucHV0czoKICAgICAgICAgICAgZm9yIHdvcmtlcl9pbnB1dCBpbiB3b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBrd2FyZ3Nbd29ya2VyX2lucHV0XQogICAgICAgICAgICAgICAgaWYgaW5wdXRfYXJndW1lbnQgaXMgTm9uZToKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgc3RyKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IF9nZXRfYXVkaW9fZmlsZXMoCiAgICAgICAgICAgICAgICAgICAgICAgIGRhdGFfcGF0aD1wYXRobGliLlBhdGgoaW5wdXRfYXJndW1lbnQpLmFic29sdXRlKCkKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBpZiBsZW4oaW5wdXRfYXJndW1lbnQpIDwgc2l6ZToKICAgICAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgICAgICAgICBmIkNhbm5vdCBzcGxpdCB0aGUgaW5wdXQgJ3t3b3JrZXJfaW5wdXR9JyBvZiBsZW5ndGgge2xlbihpbnB1dF9hcmd1bWVudCl9IHRvIHtzaXplfSB3b3JrZXJzLiAiCiAgICAgICAgICAgICAgICAgICAgICAgIGYiUGxlYXNlIHJlZHVjZSB0aGUgYW1vdW50IG9mIHdvcmtlcnMgZm9yIHRoaXMgaW5wdXQuIgogICAgICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGV2ZW5fY2h1bmtfc2l6ZSA9IGxlbihpbnB1dF9hcmd1bWVudCkgLy8gc2l6ZQogICAgICAgICAgICAgICAgY2h1bmtfc3RhcnQgPSByYW5rICogZXZlbl9jaHVua19zaXplCiAgICAgICAgICAgICAgICBjaHVua19lbmQgPSAoCiAgICAgICAgICAgICAgICAgICAgKHJhbmsgKyAxKSAqIGV2ZW5fY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgICAgIGlmIHJhbmsgKyAxIDwgc2l6ZQogICAgICAgICAgICAgICAgICAgIGVsc2UgbGVuKGlucHV0X2FyZ3VtZW50KQogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICBmIlJhbmsgI3tyYW5rfTogUHJvY2Vzc2luZyBpbnB1dCBjaHVuayBvZiAne3dvcmtlcl9pbnB1dH0nICIKICAgICAgICAgICAgICAgICAgICBmImZyb20gaW5kZXgge2NodW5rX3N0YXJ0fSB0byB7Y2h1bmtfZW5kfS4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBpZiBpc2luc3RhbmNlKGlucHV0X2FyZ3VtZW50LCBsaXN0KToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IGlucHV0X2FyZ3VtZW50W2NodW5rX3N0YXJ0OmNodW5rX2VuZF0KICAgICAgICAgICAgICAgIGVsaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgcGQuRGF0YUZyYW1lKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IGlucHV0X2FyZ3VtZW50Lmlsb2NbY2h1bmtfc3RhcnQ6Y2h1bmtfZW5kOiwgOl0KICAgICAgICAgICAgICAgIGt3YXJnc1t3b3JrZXJfaW5wdXRdID0gaW5wdXRfYXJndW1lbnQKCiAgICAgICAgICAgICMgU2V0IHRoZSByb290IHdvcmtlciBvbmx5IGFyZ3VtZW50czoKICAgICAgICAgICAgaWYgcmFuayA9PSAwIGFuZCByb290X3dvcmtlcl9pbnB1dHM6CiAgICAgICAgICAgICAgICBrd2FyZ3MudXBkYXRlKHJvb3Rfd29ya2VyX2lucHV0cykKCiAgICAgICAgICAgICMgUnVuIHRoZSB3b3JrZXI6CiAgICAgICAgICAgIG91dHB1dCA9IGhhbmRsZXIoKiprd2FyZ3MpCgogICAgICAgICAgICAjIFNlbmQgdGhlIG91dHB1dCB0byB0aGUgcm9vdCByYW5rIChyYW5rICMwKToKICAgICAgICAgICAgb3V0cHV0ID0gY29tbS5nYXRoZXIob3V0cHV0LCByb290PTApCiAgICAgICAgICAgIGlmIHJhbmsgPT0gMDoKICAgICAgICAgICAgICAgICMgSm9pbiB0aGUgb3V0cHV0czoKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkNvbGxlY3RpbmcgZGF0YSBmcm9tIHdvcmtlcnMgdG8gcm9vdCB3b3JrZXIuIikKICAgICAgICAgICAgICAgIG91dHB1dF9kaXJlY3RvcnkgPSBvdXRwdXRbMF1bMF0KICAgICAgICAgICAgICAgIGRhdGFmcmFtZSA9IHBkLmNvbmNhdChvYmpzPVtkZiBmb3IgXywgZGYsIF8gaW4gb3V0cHV0XSwgYXhpcz0wKQogICAgICAgICAgICAgICAgZXJyb3JzX2RpY3Rpb25hcnkgPSByZWR1Y2UoCiAgICAgICAgICAgICAgICAgICAgb3BlcmF0b3IuaW9yLCBbZXJyIGZvciBfLCBfLCBlcnIgaW4gb3V0cHV0XSwge30KICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJldHVybiBvdXRwdXRfZGlyZWN0b3J5LCBkYXRhZnJhbWUsIGVycm9yc19kaWN0aW9uYXJ5CiAgICAgICAgICAgIHJldHVybiBOb25lCgogICAgICAgIHJldHVybiB3cmFwcGVyCgogICAgcmV0dXJuIGRlY29yYXRvcgoKCmRlZiBfY2hlY2tfbWxydW5fYW5kX29wZW5fbXBpKCkgLT4gVHVwbGVbIm1scnVuLk1MQ2xpZW50Q3R4IiwgIm1waTRweS5NUEkuSW50cmFjb21tIl06CiAgICBpc19tcGkgPSBGYWxzZQogICAgdHJ5OgogICAgICAgIGltcG9ydCBtbHJ1bgoKICAgICAgICBjb250ZXh0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9jdHgobmFtZT0ibWxydW4iKQogICAgICAgIGlzX21waSA9IGNvbnRleHQubGFiZWxzLmdldCgia2luZCIsICJqb2IiKSA9PSAibXBpam9iIgoKICAgICAgICBpZiBpc19tcGk6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIGZyb20gbXBpNHB5IGltcG9ydCBNUEkKCiAgICAgICAgICAgICAgICByZXR1cm4gY29udGV4dCwgTVBJLkNPTU1fV09STEQKICAgICAgICAgICAgZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3IgYXMgbXBpNHB5X25vdF9mb3VuZDoKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICAgICAgICAgICJUbyBkaXN0cmlidXRlIHRoZSBmdW5jdGlvbiB1c2luZyBNTFJ1bidzICdtcGlqb2InIHlvdSBuZWVkIHRvIGhhdmUgYG1waTRweWAgcGFja2FnZSBpbiB5b3VyICIKICAgICAgICAgICAgICAgICAgICAiaW50ZXJwcmV0ZXIuIFBsZWFzZSBydW4gYHBpcCBpbnN0YWxsIG1waTRweWAgYW5kIG1ha2Ugc3VyZSB5b3UgaGF2ZSBvcGVuLW1waS4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBtcGk0cHlfbm90X2ZvdW5kCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmV0dXJuIGNvbnRleHQsIE5vbmUKICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yIGFzIG1vZHVsZV9ub3RfZm91bmQ6CiAgICAgICAgaWYgaXNfbXBpOgogICAgICAgICAgICByYWlzZSBtb2R1bGVfbm90X2ZvdW5kCiAgICByZXR1cm4gTm9uZSwgTm9uZQoKCkBvcGVuX21waV9oYW5kbGVyKHdvcmtlcl9pbnB1dHM9WyJkYXRhX3BhdGgiXSwgcm9vdF93b3JrZXJfaW5wdXRzPXsidmVyYm9zZSI6IFRydWV9KQpkZWYgdHJhbnNjcmliZSgKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBMaXN0W3N0cl1dLAogICAgb3V0cHV0X2RpcmVjdG9yeTogc3RyLAogICAgbW9kZWxfbmFtZTogc3RyID0gImJhc2UiLAogICAgZGV2aWNlOiBMaXRlcmFsWyJjdWRhIiwgImNwdSIsICJhdXRvIl0gPSAiYXV0byIsCiAgICBjb21wdXRlX3R5cGU6IHN0ciA9ICJkZWZhdWx0IiwKICAgIGxhbmd1YWdlOiBzdHIgPSBOb25lLAogICAgdHJhbnNsYXRlX3RvX2VuZ2xpc2g6IGJvb2wgPSBGYWxzZSwKICAgIHNwZWVjaF9kaWFyaXphdGlvbjogRGljdFtzdHIsIExpc3RbVHVwbGVbZmxvYXQsIGZsb2F0LCBzdHJdXV0gPSBOb25lLAogICAgYXVkaW9fZHVyYXRpb246IGJvb2wgPSBGYWxzZSwKICAgIGluaXRfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIHRyYW5zY3JpYmVfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIHZlcmJvc2U6IGJvb2wgPSBGYWxzZSwKKSAtPiBUdXBsZVtzdHIsIHBkLkRhdGFGcmFtZSwgZGljdF06CiAgICAiIiIKICAgIFRyYW5zY3JpYmUgYXVkaW8gZmlsZXMgaW50byB0ZXh0IGZpbGVzIGFuZCBjb2xsZWN0IGFkZGl0aW9uYWwgZGF0YS4gVGhlIGVuZCByZXN1bHQgaXMgYSBkaXJlY3Rvcnkgb2YgdHJhbnNjcmliZWQKICAgIHRleHQgZmlsZXMgYW5kIGEgZGF0YWZyYW1lIGNvbnRhaW5pbmcgdGhlIGZvbGxvd2luZyBjb2x1bW5zOgoKICAgICogYXVkaW9fZmlsZSAtIFRoZSBhdWRpbyBmaWxlIHBhdGguCiAgICAqIHRyYW5zY3JpcHRpb25fZmlsZSAtIFRoZSB0cmFuc2NyaWJlZCB0ZXh0IGZpbGUgbmFtZSBpbiB0aGUgb3V0cHV0IGRpcmVjdG9yeS4KICAgICogbGFuZ3VhZ2UgLSBUaGUgZGV0ZWN0ZWQgbGFuZ3VhZ2UgaW4gdGhlIGF1ZGlvIGZpbGUuCiAgICAqIGxhbmd1YWdlX3Byb2JhYmlsaXR5IC0gVGhlIGRldGVjdGVkIGxhbmd1YWdlIHByb2JhYmlsaXR5LgogICAgKiBkdXJhdGlvbiAtIFRoZSBkdXJhdGlvbiAoaW4gc2Vjb25kcykgb2YgdGhlIGF1ZGlvIGZpbGUgKG9ubHkgaWYgYGF1ZGlvX2R1cmF0aW9uYCBpcyBzZXQgdG8gVHJ1ZSkuCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgICAgICBBIGRpcmVjdG9yeSBvZiBhdWRpbyBmaWxlcyBvciBhIHNpbmdsZSBmaWxlIG9yIGEgbGlzdCBvZiBmaWxlcyB0byB0cmFuc2NyaWJlLgogICAgOnBhcmFtIG91dHB1dF9kaXJlY3Rvcnk6ICAgICAgICBQYXRoIHRvIGEgZGlyZWN0b3J5IHRvIHNhdmUgYWxsIHRyYW5zY3JpYmVkIGF1ZGlvIGZpbGVzLgogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBPbmUgb2YgdGhlIG9mZmljaWFsIG1vZGVsIG5hbWVzIG9mIFdoaXNwZXI6IHsndGlueS5lbicsICd0aW55JywgJ2Jhc2UuZW4nLCAnYmFzZScsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdzbWFsbC5lbicsICdzbWFsbCcsICdtZWRpdW0uZW4nLCAnbWVkaXVtJywgJ2xhcmdlLXYxJywgJ2xhcmdlLXYyJywgJ2xhcmdlJ30gb3IgYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmdWxsIG5hbWUgb2YgYSBmaW5lLXR1bmVkIHdoaXNwZXIgbW9kZWwgZnJvbSB0aGUgaHVnZ2luZ2ZhY2UgaHViLgogICAgOnBhcmFtIGRldmljZTogICAgICAgICAgICAgICAgICBEZXZpY2UgdG8gbG9hZCB0aGUgbW9kZWwuIENhbiBiZSBvbmUgb2YgeyJjdWRhIiwgImNwdSJ9LiBEZWZhdWx0IHdpbGwgcHJlZmVyICJjdWRhIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpZiBhdmFpbGFibGUuIFRvIHVzZSBhIHNwZWNpZmljIEdQVSBvciBtb3JlIHRoYW4gb25lIEdQVSwgcGFzcyB0aGUgYGRldmljZV9pbmRleGAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYXJndW1lbnQgdmlhIHRoZSBgaW5pdF9rd2FyZ3NgLgogICAgOnBhcmFtIGNvbXB1dGVfdHlwZTogICAgICAgICAgICBUaGUgZGF0YSB0eXBlIHRvIHVzZSBmb3IgY29tcHV0YXRpb24uIEZvciBtb3JlIGluZm9ybWF0aW9uLCBjaGVjawogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL29wZW5ubXQubmV0L0NUcmFuc2xhdGUyL3F1YW50aXphdGlvbi5odG1sLiBEZWZhdWx0OiAiZGVmYXVsdCIgLSB3aWxsIHVzZSB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGVmYXVsdCB0eXBlIGRlcGVuZGluZyBvbiB0aGUgZGV2aWNlIHVzZWQuCiAgICA6cGFyYW0gbGFuZ3VhZ2U6ICAgICAgICAgICAgICAgIFRoZSBzcG9rZW4gbGFuZ3VhZ2UgdG8gZm9yY2UgV2hpc3BlciB0aGUgb3V0cHV0IGxhbmd1YWdlLiBJZiBOb25lLCB0aGUgV2hpc3BlciBtb2RlbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGF1dG9tYXRpY2FsbHkgcHJlZGljdCB0aGUgb3V0cHV0IGxhbmdhdWdlLiBEZWZhdWx0OiBOb25lLgogICAgOnBhcmFtIHRyYW5zbGF0ZV90b19lbmdsaXNoOiAgICBXaGV0aGVyIHRvIHRyYW5zbGF0ZSB0aGUgRW5nbGlzaCBwb3N0IHRyYW5zY3JpcHRpb24uIERlZmF1bHQ6IEZhbHNlLgogICAgOnBhcmFtIHNwZWVjaF9kaWFyaXphdGlvbjogICAgICBBIHNwZWVjaCBkaWFyaXphdGlvbiBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgdG8gdHJhbnNjcmliZSBhcyBrZXlzIGFuZCB0aGVpcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkaWFyaXphdGlvbiBhcyB2YWx1ZS4gVGhlIGRpYXJpemF0aW9uIGlzIGEgbGlzdCBvZiB0dXBsZXM6IChzdGFydCwgZW5kLCBzcGVha2VyKS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgVGhlIHRyYW5zY3JpcHRpb24gcmVzdWx0IHdpbGwgYmUgaW4gdGhlIGZvbGxvd2luZyBmb3JtYXQ6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ7c3BlYWtlcn06IHRleHQgdGV4dCB0ZXh0LiIuIEZpbGVzIHdpdGggbWlzc2luZyBkaWFyaXphdGlvbnMgd2lsbCBwcmludCBhIHdhcm5pbmcuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFBheSBhdHRlbnRpb24gdGhlIGRpYXJpemF0aW9uIG11c3QgYmUgZm9yIHRoZSBlbnRpcmUgZHVyYXRpb24gb2YgdGhlIGF1ZGlvIGZpbGUgKGFzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxvbmcgYXMgV2hpc3BlciBpcyBwcmVkaWN0aW5nIHdvcmRzIHVwIHVudGlsIHRoZW4pLgogICAgOnBhcmFtIGF1ZGlvX2R1cmF0aW9uOiAgICAgICAgICBXaGV0aGVyIHRvIGluY2x1ZGUgdGhlIGF1ZGlvIGZpbGVzIGR1cmF0aW9uIChpbiBzZWNvbmRzKS4gVGhlIGVzdGltYXRlZCBkdXJhdGlvbiBpcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmcm9tIGJpdHJhdGUgYW5kIG1heSBiZSBpbmFjY3VyYXRlLiBEZWZhdWx0OiBGYWxzZS4KICAgIDpwYXJhbSBpbml0X2t3YXJnczogICAgICAgICAgICAgQWRkaXRpb25hbCBgV2hpc3Blck1vZGVsLl9faW5pdF9fYCBrZXl3b3JkIGFyZ3VtZW50cyB0byB1c2UuCiAgICA6cGFyYW0gdHJhbnNjcmliZV9rd2FyZ3M6ICAgICAgIEFkZGl0aW9uYWwgYFdoaXNwZXJNb2RlbC50cmFuc2NyaWJlYCBrZXl3b3JkIGFyZ3VtZW50cyB0byB1c2UuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gcHJlc2VudCBsb2dzIG9mIGEgcHJvZ3Jlc3MgYmFyIGFuZCBlcnJvcnMuIERlZmF1bHQ6IEZhbHNlLgoKICAgIDpyZXR1cm5zOiBBIHR1cGxlIG9mOgoKICAgICAgICAgICAgICAqIFBhdGggdG8gdGhlIG91dHB1dCBkaXJlY3RvcnkuCiAgICAgICAgICAgICAgKiBBIGRhdGFmcmFtZSBkYXRhc2V0IG9mIHRoZSB0cmFuc2NyaWJlZCBmaWxlIG5hbWVzLgogICAgICAgICAgICAgICogQSBkaWN0aW9uYXJ5IG9mIGVycm9yZWQgZmlsZXMgdGhhdCB3ZXJlIG5vdCB0cmFuc2NyaWJlZC4KICAgICIiIgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICAjIEdldCB0aGUgaW5wdXQgYXVkaW8gZmlsZXMgdG8gdHJhbnNjcmliZToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJDb2xsZWN0aW5nIGF1ZGlvIGZpbGVzLiIpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFfcGF0aCwgc3RyKToKICAgICAgICBkYXRhX3BhdGggPSBwYXRobGliLlBhdGgoZGF0YV9wYXRoKS5hYnNvbHV0ZSgpCiAgICAgICAgYXVkaW9fZmlsZXMgPSBfZ2V0X2F1ZGlvX2ZpbGVzKGRhdGFfcGF0aD1kYXRhX3BhdGgpCiAgICBlbHNlOgogICAgICAgIGF1ZGlvX2ZpbGVzID0gZGF0YV9wYXRoCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkNvbGxlY3RlZCB7bGVuKGF1ZGlvX2ZpbGVzKX0gYXVkaW8gZmlsZXMuIikKCiAgICAjIExvYWQgdGhlIHdoaXNwZXIgbW9kZWw6CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkxvYWRpbmcgbW9kZWwgJ3ttb2RlbF9uYW1lfScgLSB1c2luZyBkZXZpY2UgJ3tkZXZpY2V9Jy4iKQogICAgaW5pdF9rd2FyZ3MgPSBpbml0X2t3YXJncyBvciB7fQogICAgbW9kZWwgPSBmYXN0ZXJfd2hpc3Blci5XaGlzcGVyTW9kZWwoCiAgICAgICAgbW9kZWxfc2l6ZV9vcl9wYXRoPW1vZGVsX25hbWUsCiAgICAgICAgZGV2aWNlPWRldmljZSwKICAgICAgICBjb21wdXRlX3R5cGU9Y29tcHV0ZV90eXBlLAogICAgICAgICoqaW5pdF9rd2FyZ3MsCiAgICApCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIk1vZGVsIGxvYWRlZCBzdWNjZXNzZnVsbHkuIikKCiAgICAjIFByZXBhcmUgdGhlIHN1Y2Nlc3NlcyBkYXRhZnJhbWUgYW5kIGVycm9ycyBkaWN0aW9uYXJ5IHRvIGJlIHJldHVybmVkOgogICAgc3VjY2Vzc2VzID0gW10KICAgIGVycm9ycyA9IHt9CgogICAgIyBDcmVhdGUgdGhlIG91dHB1dCBkaXJlY3Rvcnk6CiAgICBvdXRwdXRfZGlyZWN0b3J5ID0gcGF0aGxpYi5QYXRoKG91dHB1dF9kaXJlY3RvcnkpCiAgICBvdXRwdXRfZGlyZWN0b3J5Lm1rZGlyKHBhcmVudHM9VHJ1ZSwgZXhpc3Rfb2s9VHJ1ZSkKCiAgICAjIFByZXBhcmUgdGhlIHRyYW5zY3JpYmUga2V5d29yZCBhcmd1bWVudHM6CiAgICB0cmFuc2NyaWJlX2t3YXJncyA9IHRyYW5zY3JpYmVfa3dhcmdzIG9yIHt9CiAgICB0cmFuc2NyaWJlX2t3YXJnc1sibGFuZ3VhZ2UiXSA9IGxhbmd1YWdlCiAgICB0cmFuc2NyaWJlX2t3YXJnc1sidGFzayJdID0gInRyYW5zbGF0ZSIgaWYgdHJhbnNsYXRlX3RvX2VuZ2xpc2ggZWxzZSAidHJhbnNjcmliZSIKCiAgICAjIEdvIG92ZXIgdGhlIGF1ZGlvIGZpbGVzIGFuZCB0cmFuc2NyaWJlOgogICAgZm9yIGF1ZGlvX2ZpbGUgaW4gdHFkbSgKICAgICAgICBhdWRpb19maWxlcywgZGVzYz0iVHJhbnNjcmliaW5nIiwgdW5pdD0iZmlsZSIsIGRpc2FibGU9bm90IHZlcmJvc2UKICAgICk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICAjIFRyYW5zY3JpYmU6CiAgICAgICAgICAgIHRyYW5zY3JpcHRpb25fYW5kX2luZm8gPSBfdHJhbnNjcmliZSgKICAgICAgICAgICAgICAgIGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSwKICAgICAgICAgICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgICAgICAgICAgdHJhbnNjcmliZV9rd2FyZ3M9dHJhbnNjcmliZV9rd2FyZ3MsCiAgICAgICAgICAgICAgICBzcGVlY2hfZGlhcml6YXRpb249X2dldF9kaWFyaXphdGlvbiggICMgR2V0IHRoZSBkaWFyaXphdGlvbiAoaWYgcHJvdmlkZWQpLgogICAgICAgICAgICAgICAgICAgIHNwZWVjaF9kaWFyaXphdGlvbj1zcGVlY2hfZGlhcml6YXRpb24sCiAgICAgICAgICAgICAgICAgICAgZmlsZV9uYW1lPWF1ZGlvX2ZpbGUubmFtZSwKICAgICAgICAgICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgICAgICAgICApLAogICAgICAgICAgICAgICAgYXVkaW9fZHVyYXRpb249YXVkaW9fZHVyYXRpb24sCiAgICAgICAgICAgICkKICAgICAgICAgICAgIyBXcml0ZSB0aGUgdHJhbnNjcmlwdGlvbiB0byBmaWxlOgogICAgICAgICAgICB0cmFuc2NyaXB0aW9uX2ZpbGUgPSBfc2F2ZV90b19maWxlKAogICAgICAgICAgICAgICAgdHJhbnNjcmlwdGlvbj10cmFuc2NyaXB0aW9uX2FuZF9pbmZvWzBdLAogICAgICAgICAgICAgICAgZmlsZV9uYW1lPWF1ZGlvX2ZpbGUuc3RlbSwKICAgICAgICAgICAgICAgIG91dHB1dF9kaXJlY3Rvcnk9b3V0cHV0X2RpcmVjdG9yeSwKICAgICAgICAgICAgKQogICAgICAgICAgICAjIE5vdGUgYXMgYSBzdWNjZXNzIGluIHRoZSBsaXN0OgogICAgICAgICAgICBzdWNjZXNzZXMuYXBwZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIGF1ZGlvX2ZpbGUubmFtZSwKICAgICAgICAgICAgICAgICAgICB0cmFuc2NyaXB0aW9uX2ZpbGUubmFtZSwKICAgICAgICAgICAgICAgICAgICAqdHJhbnNjcmlwdGlvbl9hbmRfaW5mb1sxOl0sCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGV4Y2VwdGlvbjoKICAgICAgICAgICAgIyBOb3RlIHRoZSBleGNlcHRpb24gYXMgZXJyb3IgaW4gdGhlIGRpY3Rpb25hcnk6CiAgICAgICAgICAgIGlmIHZlcmJvc2U6CiAgICAgICAgICAgICAgICBfTE9HR0VSLndhcm5pbmcoZiJFcnJvciBpbiBmaWxlOiAne2F1ZGlvX2ZpbGUubmFtZX0nIikKICAgICAgICAgICAgZXJyb3JzW3N0cihhdWRpb19maWxlLm5hbWUpXSA9IHN0cihleGNlcHRpb24pCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgIyBDb25zdHJ1Y3QgdGhlIHRyYW5zY3JpcHRpb25zIGRhdGFmcmFtZToKICAgIGNvbHVtbnMgPSBbCiAgICAgICAgImF1ZGlvX2ZpbGUiLAogICAgICAgICJ0cmFuc2NyaXB0aW9uX2ZpbGUiLAogICAgICAgICJsYW5ndWFnZSIsCiAgICAgICAgImxhbmd1YWdlX3Byb2JhYmlsaXR5IiwKICAgIF0KICAgIGlmIGF1ZGlvX2R1cmF0aW9uOgogICAgICAgIGNvbHVtbnMuYXBwZW5kKCJkdXJhdGlvbiIpCiAgICBzdWNjZXNzZXMgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgc3VjY2Vzc2VzLAogICAgICAgIGNvbHVtbnM9Y29sdW1ucywKICAgICkKCiAgICAjIFByaW50IHRoZSBoZWFkIG9mIHRoZSBwcm9kdWNlZCBkYXRhZnJhbWUgYW5kIHJldHVybjoKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKAogICAgICAgICAgICBmIkRvbmUgKHtzdWNjZXNzZXMuc2hhcGVbMF19L3tsZW4oYXVkaW9fZmlsZXMpfSlcbiIKICAgICAgICAgICAgZiJUcmFuc2NyaXB0aW9ucyBzdW1tYXJ5OlxuIgogICAgICAgICAgICBmIntzdWNjZXNzZXMuaGVhZCgpfSIKICAgICAgICApCiAgICByZXR1cm4gc3RyKG91dHB1dF9kaXJlY3RvcnkpLCBzdWNjZXNzZXMsIGVycm9ycwoKCmRlZiBfZ2V0X2F1ZGlvX2ZpbGVzKAogICAgZGF0YV9wYXRoOiBwYXRobGliLlBhdGgsCikgLT4gTGlzdFtwYXRobGliLlBhdGhdOgogICAgIyBDaGVjayBpZiB0aGUgcGF0aCBpcyBvZiBhIGRpcmVjdG9yeSBvciBhIGZpbGU6CiAgICBpZiBkYXRhX3BhdGguaXNfZGlyKCk6CiAgICAgICAgIyBHZXQgYWxsIGZpbGVzIGluc2lkZSB0aGUgZGlyZWN0b3J5OgogICAgICAgIGF1ZGlvX2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgYXVkaW9fZmlsZXMgPSBbZGF0YV9wYXRoXQogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlVucmVjb2duaXplZCBkYXRhIHBhdGguIFRoZSBwYXJhbWV0ZXIgYGRhdGFfcGF0aGAgbXVzdCBiZSBlaXRoZXIgYSBkaXJlY3RvcnkgcGF0aCBvciBhIGZpbGUgcGF0aC4gIgogICAgICAgICAgICBmIkdpdmVuOiB7c3RyKGRhdGFfcGF0aCl9ICIKICAgICAgICApCgogICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgoKY2xhc3MgX0RpYXJpemF0aW9uU2VnbWVudChOYW1lZFR1cGxlKToKICAgIHN0YXJ0OiBmbG9hdAogICAgZW5kOiBmbG9hdAogICAgc3BlYWtlcjogc3RyCgoKZGVmIF9nZXRfZGlhcml6YXRpb24oCiAgICBzcGVlY2hfZGlhcml6YXRpb246IERpY3Rbc3RyLCBMaXN0W1R1cGxlW2Zsb2F0LCBmbG9hdCwgc3RyXV1dLAogICAgZmlsZV9uYW1lOiBzdHIsCiAgICB2ZXJib3NlOiBib29sLAopIC0+IFVuaW9uW0xpc3RbX0RpYXJpemF0aW9uU2VnbWVudF0sIE5vbmVdOgogICAgZGlhcml6YXRpb24gPSBOb25lCiAgICBpZiBzcGVlY2hfZGlhcml6YXRpb24gaXMgbm90IE5vbmU6CiAgICAgICAgZGlhcml6YXRpb24gPSBzcGVlY2hfZGlhcml6YXRpb24uZ2V0KGZpbGVfbmFtZSkKICAgICAgICBpZiBkaWFyaXphdGlvbiBpcyBOb25lOgogICAgICAgICAgICBpZiB2ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi53YXJuaW5nKAogICAgICAgICAgICAgICAgICAgIGYiTWlzc2luZyBzcGVlY2ggZGlhcml6YXRpb24gZm9yIHRoZSBhdWRpbyBmaWxlICd7ZmlsZV9uYW1lfScuIENvbnRpbnVpbmcgdHJhbnNjcmliaW5nIHdpdGhvdXQgIgogICAgICAgICAgICAgICAgICAgIGYiZGlhcml6YXRpb24uIgogICAgICAgICAgICAgICAgKQogICAgICAgIGRpYXJpemF0aW9uID0gW19EaWFyaXphdGlvblNlZ21lbnQoKnNlZ21lbnQpIGZvciBzZWdtZW50IGluIGRpYXJpemF0aW9uXQogICAgcmV0dXJuIGRpYXJpemF0aW9uCgoKZGVmIF9nZXRfbmV4dF9kaWFyaXphdGlvbl9zZWdtZW50KAogICAgd29yZDogZmFzdGVyX3doaXNwZXIudHJhbnNjcmliZS5Xb3JkLAogICAgc3BlZWNoX2RpYXJpemF0aW9uOiBMaXN0W19EaWFyaXphdGlvblNlZ21lbnRdLAogICAgbGFzdF9jaG9zZW5faW5kZXg6IGludCwKKSAtPiBpbnQ6CiAgICAjIEdldCB0aGUgbGFzdCBjaG9zZW4gZGlhcml6YXRpb24gc2VnbWVudDoKICAgIGxhc3RfY2hvc2VuID0gc3BlZWNoX2RpYXJpemF0aW9uW2xhc3RfY2hvc2VuX2luZGV4XQoKICAgICMgSWYgdGhlIGxhc3QgY2hvc2VuIHNlZ21lbnQgaXMgdGhlIGxhc3Qgc2VnbWVudCwgcmV0dXJuIGl0OgogICAgaWYgbGFzdF9jaG9zZW5faW5kZXggPT0gbGVuKHNwZWVjaF9kaWFyaXphdGlvbikgLSAxOgogICAgICAgIHJldHVybiBsYXN0X2Nob3Nlbl9pbmRleAoKICAgICMgSWYgdGhlIHdvcmQgZW5kcyBiZWZvcmUgdGhlIGxhc3QgY2hvc2VuIHNlZ21lbnQ6CiAgICBpZiB3b3JkLmVuZCA8PSBsYXN0X2Nob3Nlbi5zdGFydDoKICAgICAgICAjIFRoZW4gaXQgaXMgc3RpbGwgdGhlIGNsb3Nlc3Qgc2VnbWVudAogICAgICAgIHJldHVybiBsYXN0X2Nob3Nlbl9pbmRleAoKICAgICMgV2UgY2hlY2sgaWYgaXQgZW5kcyBpbnNpZGUgdGhlIGxhc3QgY2hvc2VuIHNlZ21lbnQ6CiAgICBpZiB3b3JkLmVuZCA8IGxhc3RfY2hvc2VuLmVuZDoKICAgICAgICAjIFRoZW4gaXQgc3RpbGwgaXMgdGhlIGNsb3Nlc3Qgc2VnbWVudAogICAgICAgIHJldHVybiBsYXN0X2Nob3Nlbl9pbmRleAoKICAgICMgVGhlIHdvcmQgZW5kcyBhZnRlciB0aGUgc2VnbWVudCwgd2UgbmVlZCB0byBjb2xsZWN0IGFsbCBuZXh0IHNlZ21lbnRzIHVwIHVudGlsIHRoZSB3b3JkIGVuZHMgYmVmb3JlIHRoZW06CiAgICBwb3NzaWJsZV9zZWdtZW50cyA9IFtsYXN0X2Nob3Nlbl9pbmRleF0KICAgIGZvciBpIGluIHJhbmdlKGxhc3RfY2hvc2VuX2luZGV4ICsgMSwgbGVuKHNwZWVjaF9kaWFyaXphdGlvbikpOgogICAgICAgIGlmIHdvcmQuZW5kID4gc3BlZWNoX2RpYXJpemF0aW9uW2ldLmVuZDoKICAgICAgICAgICAgcG9zc2libGVfc2VnbWVudHMuYXBwZW5kKGkpCiAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgcG9zc2libGVfc2VnbWVudHMuYXBwZW5kKGkpCiAgICAgICAgYnJlYWsKCiAgICAjIENoZWNrIGZvciB0aGUgbW9zdCBvdmVybGFwcGluZyBvcHRpb246CiAgICBiZXN0X292ZXJsYXAgPSAwCiAgICBvdmVybGFwcGluZ19zZWdtZW50ID0gTm9uZQogICAgZm9yIGkgaW4gcG9zc2libGVfc2VnbWVudHM6CiAgICAgICAgb3ZlcmxhcCA9IDAKICAgICAgICAjIElmIHRoZSB3b3JkIHN0YXJ0cyBiZWZvcmUgc2VnbWVudDoKICAgICAgICBpZiB3b3JkLnN0YXJ0IDw9IHNwZWVjaF9kaWFyaXphdGlvbltpXS5zdGFydDoKICAgICAgICAgICAgIyBJZiBpdCBlbmRzIGJlZm9yZSB0aGUgc2VnbWVudCwgdGhlcmUgaXMgYW4gb3ZlcmxhcCBmcm9tIHRoZSBzdGFydCBvZiB0aGUgc2VnbWVudCB0byB0aGUgZW5kIG9mIHRoZSB3b3JkOgogICAgICAgICAgICBpZiB3b3JkLmVuZCA8IHNwZWVjaF9kaWFyaXphdGlvbltpXS5lbmQ6CiAgICAgICAgICAgICAgICBvdmVybGFwID0gd29yZC5lbmQgLSBzcGVlY2hfZGlhcml6YXRpb25baV0uc3RhcnQKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgICMgVGhlIHdvcmQgaXMgd3JhcHBpbmcgdGhlIHNlZ21lbnQsIHRoZSBvdmVybGFwIGlzIHRoZSBzZWdtZW50J3MgbGVuZ3RoOgogICAgICAgICAgICAgICAgb3ZlcmxhcCA9IHNwZWVjaF9kaWFyaXphdGlvbltpXS5lbmQgLSBzcGVlY2hfZGlhcml6YXRpb25baV0uc3RhcnQKICAgICAgICAjIFRoZSB3b3JkIHN0YXJ0cyBpbiBzZWdtZW50LCBjaGVjayBpZiB0aGUgd29yZCBlbmRzIGluIGl0OgogICAgICAgIGVsaWYgd29yZC5lbmQgPCBzcGVlY2hfZGlhcml6YXRpb25baV0uZW5kOgogICAgICAgICAgICAjIFRoZSBvdmVybGFwIGlzIHRoZSB3b3JkJ3MgbGVuZ3RoOgogICAgICAgICAgICBvdmVybGFwID0gd29yZC5lbmQgLSB3b3JkLnN0YXJ0CiAgICAgICAgIyBUaGUgd29yZCBzdGFydCBpbiBzZWdtZW50IGJ1dCBlbmRzIGFmdGVyIGl0LCB0aGUgb3ZlcmxhcCBpcyBmcm9tIHRoZSB3b3JkJ3Mgc3RhcnQgdG8gdGhlIHNlZ21lbnQncyBlbmQ6CiAgICAgICAgZWxzZToKICAgICAgICAgICAgb3ZlcmxhcCA9IHNwZWVjaF9kaWFyaXphdGlvbltpXS5lbmQgLSB3b3JkLnN0YXJ0CiAgICAgICAgIyBDaGVjayBmb3IgbmV3IGJlc3Qgb3ZlcmxhcDoKICAgICAgICBpZiBvdmVybGFwID4gYmVzdF9vdmVybGFwOgogICAgICAgICAgICBiZXN0X292ZXJsYXAgPSBvdmVybGFwCiAgICAgICAgICAgIG92ZXJsYXBwaW5nX3NlZ21lbnQgPSBpCiAgICBpZiBvdmVybGFwcGluZ19zZWdtZW50IGlzIG5vdCBOb25lOgogICAgICAgIHJldHVybiBvdmVybGFwcGluZ19zZWdtZW50CgogICAgIyBJZiB0aGVyZSBpcyBubyBvdmVybGFwcGluZyBzZWdtZW50LCByZXR1cm4gdGhlIGNsb3Nlc3Qgc2VnbWVudDoKICAgIGJlc3RfZGlzdGFuY2UgPSBOb25lCiAgICBjbG9zZXN0X3NlZ21lbnQgPSBOb25lCiAgICBmb3IgaSBpbiBwb3NzaWJsZV9zZWdtZW50czoKICAgICAgICBkaXN0YW5jZSA9ICgKICAgICAgICAgICAgd29yZC5zdGFydCAtIHNwZWVjaF9kaWFyaXphdGlvbltpXS5lbmQKICAgICAgICAgICAgaWYgd29yZC5zdGFydCA+IHNwZWVjaF9kaWFyaXphdGlvbltpXS5lbmQKICAgICAgICAgICAgZWxzZSBzcGVlY2hfZGlhcml6YXRpb25baV0uc3RhcnQgLSB3b3JkLmVuZAogICAgICAgICkKICAgICAgICBpZiBiZXN0X2Rpc3RhbmNlIGlzIE5vbmUgb3IgZGlzdGFuY2UgPCBiZXN0X2Rpc3RhbmNlOgogICAgICAgICAgICBiZXN0X2Rpc3RhbmNlID0gZGlzdGFuY2UKICAgICAgICAgICAgY2xvc2VzdF9zZWdtZW50ID0gaQogICAgcmV0dXJuIGNsb3Nlc3Rfc2VnbWVudAoKCmRlZiBfY29uc3RydWN0X3RyYW5zY3JpcHRpb24oCiAgICBzZWdtZW50czogTGlzdFtmYXN0ZXJfd2hpc3Blci50cmFuc2NyaWJlLlNlZ21lbnRdLAogICAgc3BlZWNoX2RpYXJpemF0aW9uOiBMaXN0W19EaWFyaXphdGlvblNlZ21lbnRdLAopIC0+IHN0cjoKICAgICMgSWYgdGhlcmUgaXMgbm8gZGlhcml6YXRpb24sIGNvbmNhdGVuYXRlIGFsbCBzZWdtZW50cyBhbmQgcmV0dXJuOgogICAgaWYgc3BlZWNoX2RpYXJpemF0aW9uIGlzIE5vbmU6CiAgICAgICAgcmV0dXJuICIgIi5qb2luKFtzZWdtZW50LnRleHQgZm9yIHNlZ21lbnQgaW4gc2VnbWVudHNdKQoKICAgICMgVGhlcmUgaXMgYSBkaWFyaXphdGlvbiwgdHJ5IHRvIG1hdGNoIHRoZSBXaGlzcGVyIG1vZGVsIHByZWRpY3RlZCB0aW1lc3RhbXBzIHRvIHRoZSBjbG9zZXN0IGRpYXJpemF0aW9uIHNlZ21lbnQKICAgICMgKGNsb3Nlc3QgZGlhcml6YXRpb24gc2VnbWVudCB3aWxsIGJlIHRoZSBtb3N0IG92ZXJsYXBwaW5nIHdpdGggdGhlIHdvcmQsIGFuZCBpZiB0aGVyZSBpcyBubyBvdmVybGFwLCB0aGUgY2xvc2VzdAogICAgIyBzZWdtZW50IHRvIHRoZSB3b3JkKToKICAgIGRpYXJpemF0aW9uX2luZGV4ID0gMAogICAgc3BlYWtlciA9IHNwZWVjaF9kaWFyaXphdGlvbltkaWFyaXphdGlvbl9pbmRleF0uc3BlYWtlcgogICAgdGV4dCA9IGYie3NwZWFrZXJ9OiIKICAgIGZvciBzZWdtZW50IGluIHNlZ21lbnRzOgogICAgICAgIGZvciB3b3JkIGluIHNlZ21lbnQud29yZHM6CiAgICAgICAgICAgICMgR2V0IHRoZSBuZXh0IGRpYXJpemF0aW9uIHNlZ21lbnQ6CiAgICAgICAgICAgIGRpYXJpemF0aW9uX2luZGV4ID0gX2dldF9uZXh0X2RpYXJpemF0aW9uX3NlZ21lbnQoCiAgICAgICAgICAgICAgICB3b3JkPXdvcmQsCiAgICAgICAgICAgICAgICBzcGVlY2hfZGlhcml6YXRpb249c3BlZWNoX2RpYXJpemF0aW9uLAogICAgICAgICAgICAgICAgbGFzdF9jaG9zZW5faW5kZXg9ZGlhcml6YXRpb25faW5kZXgsCiAgICAgICAgICAgICkKICAgICAgICAgICAgIyBDaGVjayBpZiB0aGUgc2VnbWVudCBpcyBvZiB0aGUgc2FtZSBzcGVha2VyOgogICAgICAgICAgICBpZiBzcGVlY2hfZGlhcml6YXRpb25bZGlhcml6YXRpb25faW5kZXhdLnNwZWFrZXIgPT0gc3BlYWtlcjoKICAgICAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgd29yZDoKICAgICAgICAgICAgICAgIHRleHQgKz0gd29yZC53b3JkCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAjIEFwcGVuZCBhIG5ld2xpbmUgYW5kIHVwZGF0ZSB0aGUgbmV3IHNwZWFrZXI6CiAgICAgICAgICAgICAgICBzcGVha2VyID0gc3BlZWNoX2RpYXJpemF0aW9uW2RpYXJpemF0aW9uX2luZGV4XS5zcGVha2VyCiAgICAgICAgICAgICAgICB0ZXh0ICs9IGYiXG57c3BlYWtlcn06e3dvcmQud29yZH0iCgogICAgcmV0dXJuIHRleHQKCgpkZWYgX3RyYW5zY3JpYmUoCiAgICBhdWRpb19maWxlOiBwYXRobGliLlBhdGgsCiAgICBtb2RlbDogZmFzdGVyX3doaXNwZXIuV2hpc3Blck1vZGVsLAogICAgdHJhbnNjcmliZV9rd2FyZ3M6IGRpY3QsCiAgICBzcGVlY2hfZGlhcml6YXRpb246IExpc3RbX0RpYXJpemF0aW9uU2VnbWVudF0sCiAgICBhdWRpb19kdXJhdGlvbjogYm9vbCwKKSAtPiBVbmlvbltUdXBsZVtzdHIsIHN0ciwgZmxvYXRdLCBUdXBsZVtzdHIsIHN0ciwgZmxvYXQsIGZsb2F0XV06CiAgICAjIFRyYW5zY3JpYmUgKFNlZ21lbnRzIGlzIGEgZ2VuZXJhdG9yLCBzbyB3ZSBjYXN0IHRvIGxpc3QgdG8gYmVnaW4gdHJhbnNjcmlwdGlvbiBmcm9tIHN0YXJ0IHRvIGVuZCk6CiAgICBzZWdtZW50cywgaW5mbyA9IG1vZGVsLnRyYW5zY3JpYmUoCiAgICAgICAgYXVkaW89c3RyKGF1ZGlvX2ZpbGUpLAogICAgICAgICoqdHJhbnNjcmliZV9rd2FyZ3MsCiAgICAgICAgd29yZF90aW1lc3RhbXBzPXNwZWVjaF9kaWFyaXphdGlvbiBpcyBub3QgTm9uZSwKICAgICkKICAgIHNlZ21lbnRzID0gbGlzdChzZWdtZW50cykKCiAgICAjIENoZWNrIGlmIHNwZWVjaCBkaWFyaXphdGlvbiB3YXMgcHJvdmlkZWQ6CiAgICBpZiBzcGVlY2hfZGlhcml6YXRpb24gaXMgTm9uZToKICAgICAgICB0ZXh0ID0gIiIuam9pbihbc2VnbWVudC50ZXh0IGZvciBzZWdtZW50IGluIHNlZ21lbnRzXSkKICAgIGVsc2U6CiAgICAgICAgdGV4dCA9IF9jb25zdHJ1Y3RfdHJhbnNjcmlwdGlvbigKICAgICAgICAgICAgc2VnbWVudHM9c2VnbWVudHMsCiAgICAgICAgICAgIHNwZWVjaF9kaWFyaXphdGlvbj1zcGVlY2hfZGlhcml6YXRpb24sCiAgICAgICAgKQogICAgdGV4dCA9IHRleHQuc3RyaXAoKQoKICAgICMgUmV0dXJuIHRoZSB0cmFuc2NyaXB0aW9uIHRleHQgYW5kIHRoZSBhZGRpdGlvbmFsIGluZm9ybWF0aW9uOgogICAgaWYgYXVkaW9fZHVyYXRpb246CiAgICAgICAgcmV0dXJuIHRleHQuc3RyaXAoKSwgaW5mby5sYW5ndWFnZSwgaW5mby5sYW5ndWFnZV9wcm9iYWJpbGl0eSwgaW5mby5kdXJhdGlvbgogICAgcmV0dXJuIHRleHQuc3RyaXAoKSwgaW5mby5sYW5ndWFnZSwgaW5mby5sYW5ndWFnZV9wcm9iYWJpbGl0eQoKCmRlZiBfc2F2ZV90b19maWxlKAogICAgdHJhbnNjcmlwdGlvbjogc3RyLCBmaWxlX25hbWU6IHN0ciwgb3V0cHV0X2RpcmVjdG9yeTogcGF0aGxpYi5QYXRoCikgLT4gcGF0aGxpYi5QYXRoOgogICAgIyBQcmVwYXJlIHRoZSBmaWxlIGZ1bGwgcGF0aCAoY2hlY2tpbmcgZm9yIG5vIGR1cGxpY2F0aW9ucyk6CiAgICB0cmFuc2NyaXB0aW9uX2ZpbGUgPSBvdXRwdXRfZGlyZWN0b3J5IC8gZiJ7ZmlsZV9uYW1lfS50eHQiCiAgICBpID0gMQogICAgd2hpbGUgdHJhbnNjcmlwdGlvbl9maWxlLmV4aXN0cygpOgogICAgICAgIGkgKz0gMQogICAgICAgIHRyYW5zY3JpcHRpb25fZmlsZSA9IG91dHB1dF9kaXJlY3RvcnkgLyBmIntmaWxlX25hbWV9X3tpfS50eHQiCgogICAgIyBNYWtlIHN1cmUgYWxsIGRpcmVjdG9yaWVzIGFyZSBjcmVhdGVkOgogICAgdHJhbnNjcmlwdGlvbl9maWxlLnBhcmVudC5ta2RpcihleGlzdF9vaz1UcnVlLCBwYXJlbnRzPVRydWUpCgogICAgIyBXcml0ZSB0byBmaWxlOgogICAgd2l0aCBvcGVuKHRyYW5zY3JpcHRpb25fZmlsZSwgInciKSBhcyBmcDoKICAgICAgICBmcC53cml0ZSh0cmFuc2NyaXB0aW9uKQoKICAgIHJldHVybiB0cmFuc2NyaXB0aW9uX2ZpbGUK + functionSourceCode: IyBDb3B5cmlnaHQgMjAyNCBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgbG9nZ2luZwppbXBvcnQgb3BlcmF0b3IKaW1wb3J0IG9zCmltcG9ydCB0ZW1wZmlsZQpmcm9tIGZ1bmN0b29scyBpbXBvcnQgcmVkdWNlLCB3cmFwcwpmcm9tIG11bHRpcHJvY2Vzc2luZyBpbXBvcnQgUHJvY2VzcywgUXVldWUKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIEdlbmVyYXRvciwgTGlzdCwgTGl0ZXJhbCwgTmFtZWRUdXBsZSwgVHVwbGUsIFVuaW9uCgppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCB0b3JjaAppbXBvcnQgdG9yY2hhdWRpbwpmcm9tIHRxZG0gaW1wb3J0IHRxZG0KZnJvbSB0cmFuc2Zvcm1lcnMgaW1wb3J0ICgKICAgIEF1dG9tYXRpY1NwZWVjaFJlY29nbml0aW9uUGlwZWxpbmUsCiAgICBBdXRvTW9kZWxGb3JDYXVzYWxMTSwKICAgIHBpcGVsaW5lLAopCmZyb20gdHJhbnNmb3JtZXJzLnV0aWxzIGltcG9ydCBpc19mbGFzaF9hdHRuXzJfYXZhaWxhYmxlCgoKY2xhc3MgQmFzZVRhc2s6CiAgICAiIiIKICAgIEEgdGFzayB0byB3cml0ZSB0aGUgdHJhbnNjcmlwdGlvbiB0byBmaWxlLgogICAgIiIiCgogICAgZGVmIF9faW5pdF9fKAogICAgICAgIHNlbGYsIGF1ZGlvX2ZpbGU6IFBhdGgsIHRyYW5zY3JpcHRpb25fb3V0cHV0OiBVbmlvbltkaWN0LCBzdHJdLCB0ZXh0X2ZpbGU6IFBhdGgKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgSW5pdGlhbGl6ZSB0aGUgdGFzay4KCiAgICAgICAgOnBhcmFtIGF1ZGlvX2ZpbGU6ICAgICAgICAgICBQYXRoIHRvIHRoZSBhdWRpbyBmaWxlIHRoYXQgd2FzIHRyYW5zY3JpYmVkLgogICAgICAgIDpwYXJhbSB0cmFuc2NyaXB0aW9uX291dHB1dDogVGhlIHRyYW5zY3JpcHRpb24gb3V0cHV0IGZyb20gdGhlIHBpcGVsaW5lLiBTdHJpbmcgbWVhbnMgYW4gZXhjZXB0aW9uIHdhcyByYWlzZWQuCiAgICAgICAgOnBhcmFtIHRleHRfZmlsZTogICAgICAgICAgICBQYXRoIHRvIHRoZSB0ZXh0IGZpbGUgdG8gd3JpdGUgdGhlIHRyYW5zY3JpcHRpb24gdG8uCiAgICAgICAgIiIiCiAgICAgICAgIyBTdG9yZSB0aGUgcGFyYW1ldGVyczoKICAgICAgICBzZWxmLl9hdWRpb19maWxlID0gYXVkaW9fZmlsZQogICAgICAgIHNlbGYuX3RyYW5zY3JpcHRpb25fb3V0cHV0ID0gdHJhbnNjcmlwdGlvbl9vdXRwdXQKICAgICAgICBzZWxmLl90ZXh0X2ZpbGUgPSB0ZXh0X2ZpbGUKCiAgICAgICAgIyBQcmVwYXJlIHRoZSBlcnJvciB2YXJpYWJsZToKICAgICAgICBzZWxmLl9lcnJvcjogc3RyID0gTm9uZQoKICAgIGRlZiBkb190YXNrKHNlbGYpOgogICAgICAgICIiIgogICAgICAgIFRyeSB0byBwZXJmb3JtIHRoZSB0YXNrIHN0b3JpbmcgYW4gZXJyb3IgaWYgb2NjdXJyZWQuCiAgICAgICAgIiIiCiAgICAgICAgaWYgaXNpbnN0YW5jZShzZWxmLl90cmFuc2NyaXB0aW9uX291dHB1dCwgc3RyKToKICAgICAgICAgICAgc2VsZi5fZXJyb3IgPSBzZWxmLl90cmFuc2NyaXB0aW9uX291dHB1dAogICAgICAgICAgICByZXR1cm4KICAgICAgICB0cnk6CiAgICAgICAgICAgIHNlbGYuX2RvX3Rhc2soKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICBzZWxmLl9lcnJvciA9IHN0cihleGNlcHRpb24pCgogICAgZGVmIGlzX2ZhaWxlZChzZWxmKSAtPiBib29sOgogICAgICAgICIiIgogICAgICAgIENoZWNrIGlmIHRoZSB0YXNrIGZhaWxlZC4KCiAgICAgICAgOnJldHVybnM6IFdoZXRoZXIgdGhlIHRhc2sgZmFpbGVkLgogICAgICAgICIiIgogICAgICAgIHJldHVybiBzZWxmLl9lcnJvciBpcyBub3QgTm9uZQoKICAgIGRlZiBnZXRfcmVzdWx0KHNlbGYpIC0+IFR1cGxlW3N0ciwgc3RyXToKICAgICAgICAiIiIKICAgICAgICBHZXQgdGhlIHJlc3VsdCBvZiB0aGUgdGFzay4gSWYgdGhlIHRhc2sgZmFpbGVkLCB0aGUgZXJyb3Igd2lsbCBiZSByZXR1cm5lZCwgb3RoZXJ3aXNlLCB0aGUgcmVzdWx0IHdpbGwgYmUgdGhlCiAgICAgICAgdGV4dCBmaWxlIG5hbWUuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgdGFzaydzIHJlc3VsdC4KICAgICAgICAiIiIKICAgICAgICBpZiBzZWxmLmlzX2ZhaWxlZCgpOgogICAgICAgICAgICByZXR1cm4gc2VsZi5fYXVkaW9fZmlsZS5uYW1lLCBzZWxmLl9lcnJvcgogICAgICAgIHJldHVybiBzZWxmLl9hdWRpb19maWxlLm5hbWUsIHNlbGYuX3RleHRfZmlsZS5uYW1lCgogICAgZGVmIHRvX3R1cGxlKHNlbGYpIC0+IFR1cGxlW3N0ciwgZGljdF06CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCB0aGUgdGFzayB0byBhIHR1cGxlIHRvIHJlY29uc3RydWN0IGl0IGxhdGVyICh1c2VkIGZvciBtdWx0aXByb2Nlc3NpbmcgdG8gcGFzcyBpbiBxdWV1ZSkuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgY29udmVydGVkIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX19jbGFzc19fLl9fbmFtZV9fLCB7CiAgICAgICAgICAgICJhdWRpb19maWxlIjogc2VsZi5fYXVkaW9fZmlsZSwKICAgICAgICAgICAgInRyYW5zY3JpcHRpb25fb3V0cHV0Ijogc2VsZi5fdHJhbnNjcmlwdGlvbl9vdXRwdXQsCiAgICAgICAgICAgICJ0ZXh0X2ZpbGUiOiBzZWxmLl90ZXh0X2ZpbGUsCiAgICAgICAgfQoKICAgIGRlZiBfZG9fdGFzayhzZWxmKToKICAgICAgICAiIiIKICAgICAgICBQZXJmb3JtIHRoZSB0YXNrIC0gd3JpdGUgdGhlIHRyYW5zY3JpcHRpb24gdG8gdGhlIHN0b3JlZCBmaWxlIHBhdGguCiAgICAgICAgIiIiCiAgICAgICAgIyBDaGVja2luZyBmb3Igbm8gZHVwbGljYXRpb25zOgogICAgICAgIGkgPSAxCiAgICAgICAgd2hpbGUgc2VsZi5fdGV4dF9maWxlLmV4aXN0cygpOgogICAgICAgICAgICBpICs9IDEKICAgICAgICAgICAgc2VsZi5fdGV4dF9maWxlID0gKAogICAgICAgICAgICAgICAgc2VsZi5fdGV4dF9maWxlLnBhcmVudAogICAgICAgICAgICAgICAgLyBmIntzZWxmLl90ZXh0X2ZpbGUuc3RlbS5yc3BsaXQoJ18nLCAxKVswXX1fe2l9e3NlbGYuX3RleHRfZmlsZS5zdWZmaXh9IgogICAgICAgICAgICApCgogICAgICAgICMgTWFrZSBzdXJlIGFsbCBkaXJlY3RvcmllcyBhcmUgY3JlYXRlZDoKICAgICAgICBzZWxmLl90ZXh0X2ZpbGUucGFyZW50Lm1rZGlyKGV4aXN0X29rPVRydWUsIHBhcmVudHM9VHJ1ZSkKCiAgICAgICAgIyBXcml0ZSB0byBmaWxlOgogICAgICAgIHdpdGggb3BlbihzZWxmLl90ZXh0X2ZpbGUsICJ3IikgYXMgZnA6CiAgICAgICAgICAgIGZwLndyaXRlKHNlbGYuX3RyYW5zY3JpcHRpb25fb3V0cHV0WyJ0ZXh0Il0pCgoKY2xhc3MgU3BlZWNoRGlhcml6YXRpb25UYXNrKEJhc2VUYXNrKToKICAgICIiIgogICAgQSB0YXNrIHRvIHdyaXRlIHRoZSB0cmFuc2NyaXB0aW9uIHRvIGZpbGUgd2l0aCByZXNwZWN0IHRvIGEgZ2l2ZW4gc3BlZWNoIGRpYXJpemF0aW9uLgogICAgIiIiCgogICAgY2xhc3MgX0RpYXJpemF0aW9uU2VnbWVudChOYW1lZFR1cGxlKToKICAgICAgICAiIiIKICAgICAgICBBIHNwZWVjaCBkaWFyaXphdGlvbiBzZWdtZW50LgogICAgICAgICIiIgoKICAgICAgICBzdGFydDogZmxvYXQKICAgICAgICBlbmQ6IGZsb2F0CiAgICAgICAgc3BlYWtlcjogc3RyCgogICAgY2xhc3MgX1dvcmRUaW1lc3RhbXAoTmFtZWRUdXBsZSk6CiAgICAgICAgIiIiCiAgICAgICAgQSB3b3JkIHdpdGggaXRzIHN0YXJ0IGFuZCBlbmQgdGltZXN0YW1wcy4KICAgICAgICAiIiIKCiAgICAgICAgc3RhcnQ6IGZsb2F0CiAgICAgICAgZW5kOiBmbG9hdAogICAgICAgIHRleHQ6IHN0cgoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgIGF1ZGlvX2ZpbGU6IFBhdGgsCiAgICAgICAgdHJhbnNjcmlwdGlvbl9vdXRwdXQ6IGRpY3QsCiAgICAgICAgdGV4dF9maWxlOiBQYXRoLAogICAgICAgIHNwZWVjaF9kaWFyaXphdGlvbjogTGlzdFtUdXBsZVtmbG9hdCwgZmxvYXQsIHN0cl1dLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSB0YXNrLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogICAgICAgICAgIFBhdGggdG8gdGhlIGF1ZGlvIGZpbGUgdGhhdCB3YXMgdHJhbnNjcmliZWQuCiAgICAgICAgOnBhcmFtIHRyYW5zY3JpcHRpb25fb3V0cHV0OiBUaGUgdHJhbnNjcmlwdGlvbiBvdXRwdXQgZnJvbSB0aGUgcGlwZWxpbmUuCiAgICAgICAgOnBhcmFtIHRleHRfZmlsZTogICAgICAgICAgICBQYXRoIHRvIHRoZSB0ZXh0IGZpbGUgdG8gd3JpdGUgdGhlIHRyYW5zY3JpcHRpb24gdG8uCiAgICAgICAgOnBhcmFtIHNwZWVjaF9kaWFyaXphdGlvbjogICBBIHNwZWVjaCBkaWFyaXphdGlvbiBhcyBhIGxpc3Qgb2YgdHVwbGVzOiAoc3RhcnQsIGVuZCwgc3BlYWtlcikuCiAgICAgICAgIiIiCiAgICAgICAgc3VwZXIoKS5fX2luaXRfXygKICAgICAgICAgICAgYXVkaW9fZmlsZT1hdWRpb19maWxlLAogICAgICAgICAgICB0cmFuc2NyaXB0aW9uX291dHB1dD10cmFuc2NyaXB0aW9uX291dHB1dCwKICAgICAgICAgICAgdGV4dF9maWxlPXRleHRfZmlsZSwKICAgICAgICApCiAgICAgICAgc2VsZi5fc3BlZWNoX2RpYXJpemF0aW9uID0gc3BlZWNoX2RpYXJpemF0aW9uCiAgICAgICAgc2VsZi5fc2VnbWVudHM6IExpc3RbU3BlZWNoRGlhcml6YXRpb25UYXNrLl9EaWFyaXphdGlvblNlZ21lbnRdID0gTm9uZQogICAgICAgIHNlbGYuX2xhc3RfY2hvc2VuX2luZGV4ID0gMAoKICAgIGRlZiB0b190dXBsZShzZWxmKSAtPiBUdXBsZVtzdHIsIGRpY3RdOgogICAgICAgICIiIgogICAgICAgIENvbnZlcnQgdGhlIHRhc2sgdG8gYSB0dXBsZSB0byByZWNvbnN0cnVjdCBpdCBsYXRlciAodXNlZCBmb3IgbXVsdGlwcm9jZXNzaW5nIHRvIHBhc3MgaW4gcXVldWUpLgoKICAgICAgICA6cmV0dXJuczogVGhlIGNvbnZlcnRlZCB0YXNrLgogICAgICAgICIiIgogICAgICAgIHRhc2tfY2xhc3MsIHRhc2tfa3dhcmdzID0gc3VwZXIoKS50b190dXBsZSgpCiAgICAgICAgcmV0dXJuIHRhc2tfY2xhc3MsIHsKICAgICAgICAgICAgKip0YXNrX2t3YXJncywKICAgICAgICAgICAgInNwZWVjaF9kaWFyaXphdGlvbiI6IHNlbGYuX3NwZWVjaF9kaWFyaXphdGlvbiwKICAgICAgICB9CgogICAgZGVmIF9kb190YXNrKHNlbGYpOgogICAgICAgICIiIgogICAgICAgIFBlcmZvcm0gdGhlIHRhc2sgLSB3cml0ZSB0aGUgdHJhbnNjcmlwdGlvbiB0byB0aGUgc3RvcmVkIGZpbGUgcGF0aCB3aXRoIHJlc3BlY3QgdG8gdGhlIGdpdmVuIHNwZWVjaCBkaWFyaXphdGlvbi4KICAgICAgICAiIiIKICAgICAgICAjIENoZWNrIGlmIGEgc3BlZWNoIGRpYXJpemF0aW9uIGlzIGdpdmVuLCBpZiBub3QsIGp1c3Qgd3JpdGUgdGhlIHRyYW5zY3JpcHRpb24gdG8gZmlsZToKICAgICAgICBpZiBub3Qgc2VsZi5fc3BlZWNoX2RpYXJpemF0aW9uOgogICAgICAgICAgICBzdXBlcigpLl9kb190YXNrKCkKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgQ2FzdCB0aGUgY2h1bmtzIHRvIHdvcmQgdGltZXN0YW1wcyB0dXBsZXM6CiAgICAgICAgd29yZHMgPSBbCiAgICAgICAgICAgIFNwZWVjaERpYXJpemF0aW9uVGFzay5fV29yZFRpbWVzdGFtcCgKICAgICAgICAgICAgICAgIHN0YXJ0PWNodW5rWyJ0aW1lc3RhbXAiXVswXSwKICAgICAgICAgICAgICAgIGVuZD1jaHVua1sidGltZXN0YW1wIl1bMV0sCiAgICAgICAgICAgICAgICB0ZXh0PWNodW5rWyJ0ZXh0Il0sCiAgICAgICAgICAgICkKICAgICAgICAgICAgZm9yIGNodW5rIGluIHNlbGYuX3RyYW5zY3JpcHRpb25fb3V0cHV0WyJjaHVua3MiXQogICAgICAgIF0KCiAgICAgICAgIyBDYXN0IHNwZWVjaCBkaWFyaXphdGlvbiB0byBzZWdtZW50cyB0dXBsZXM6CiAgICAgICAgc2VsZi5fc2VnbWVudHMgPSBbCiAgICAgICAgICAgIFNwZWVjaERpYXJpemF0aW9uVGFzay5fRGlhcml6YXRpb25TZWdtZW50KCpzZWdtZW50KQogICAgICAgICAgICBmb3Igc2VnbWVudCBpbiBzZWxmLl9zcGVlY2hfZGlhcml6YXRpb24KICAgICAgICBdCgogICAgICAgICMgVHJ5IHRvIG1hdGNoIHRoZSBXaGlzcGVyIG1vZGVsIHByZWRpY3RlZCB0aW1lc3RhbXBzIHRvIHRoZSBjbG9zZXN0IGRpYXJpemF0aW9uIHNlZ21lbnQgKGNsb3Nlc3QgZGlhcml6YXRpb24KICAgICAgICAjIHNlZ21lbnQgd2lsbCBiZSB0aGUgbW9zdCBvdmVybGFwcGluZyB3aXRoIHRoZSB3b3JkLCBhbmQgaWYgdGhlcmUgaXMgbm8gb3ZlcmxhcCwgdGhlIGNsb3Nlc3Qgc2VnbWVudCB0byB0aGUKICAgICAgICAjIHdvcmQpOgogICAgICAgIHNwZWFrZXIgPSBzZWxmLl9zZWdtZW50c1tzZWxmLl9sYXN0X2Nob3Nlbl9pbmRleF0uc3BlYWtlcgogICAgICAgIHRleHQgPSBmIntzcGVha2VyfToiCiAgICAgICAgZm9yIHdvcmQgaW4gd29yZHM6CiAgICAgICAgICAgICMgR2V0IHRoZSBuZXh0IGRpYXJpemF0aW9uIHNlZ21lbnQ6CiAgICAgICAgICAgIHNlbGYuX2dldF9uZXh0X3NlZ21lbnQod29yZD13b3JkKQogICAgICAgICAgICAjIENoZWNrIGlmIHRoZSBzZWdtZW50IGlzIG9mIHRoZSBzYW1lIHNwZWFrZXI6CiAgICAgICAgICAgIGlmIHNlbGYuX3NlZ21lbnRzW3NlbGYuX2xhc3RfY2hvc2VuX2luZGV4XS5zcGVha2VyID09IHNwZWFrZXI6CiAgICAgICAgICAgICAgICAjIENvbGxlY3QgdGhlIHdvcmQ6CiAgICAgICAgICAgICAgICB0ZXh0ICs9IHdvcmQudGV4dAogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgIyBBcHBlbmQgYSBuZXdsaW5lIGFuZCB1cGRhdGUgdGhlIG5ldyBzcGVha2VyOgogICAgICAgICAgICAgICAgc3BlYWtlciA9IHNlbGYuX3NlZ21lbnRzW3NlbGYuX2xhc3RfY2hvc2VuX2luZGV4XS5zcGVha2VyCiAgICAgICAgICAgICAgICB0ZXh0ICs9IGYiXG57c3BlYWtlcn06e3dvcmQudGV4dH0iCgogICAgICAgICMgVXBkYXRlIHRoZSB0cmFuc2NyaXB0aW9uIG91dHB1dCB3aXRoIHRoZSBuZXcgdGV4dCB0byB3cml0ZSBpdCB0byBmaWxlOgogICAgICAgIHNlbGYuX3RyYW5zY3JpcHRpb25fb3V0cHV0WyJ0ZXh0Il0gPSB0ZXh0CiAgICAgICAgc3VwZXIoKS5fZG9fdGFzaygpCgogICAgZGVmIF9nZXRfbmV4dF9zZWdtZW50KAogICAgICAgIHNlbGYsCiAgICAgICAgd29yZDogX1dvcmRUaW1lc3RhbXAsCiAgICApOgogICAgICAgICIiIgogICAgICAgIEdldCB0aGUgbmV4dCBkaWFyaXphdGlvbiBzZWdtZW50IHRoZSBnaXZlbiB3b3JkIGZhbGxzIGludG8uIFRoZSBgc2VsZi5fbGFzdF9jaG9zZW5faW5kZXhgIHdpbGwgYmUgdXBkYXRlZAogICAgICAgIGFjY29yZGluZ2x5LgoKICAgICAgICA6cGFyYW0gd29yZDogVGhlIHdvcmQgdGltZXN0YW1wIHRvIG1hdGNoIHRvIHRoZSBuZXh0IHNlZ21lbnQuCiAgICAgICAgIiIiCiAgICAgICAgIyBJZiB0aGUgbGFzdCBjaG9zZW4gc2VnbWVudCBpcyB0aGUgbGFzdCBzZWdtZW50LCByZXR1cm4gaXQ6CiAgICAgICAgaWYgc2VsZi5fbGFzdF9jaG9zZW5faW5kZXggPT0gbGVuKHNlbGYuX3NlZ21lbnRzKSAtIDE6CiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjIEdldCB0aGUgbGFzdCBjaG9zZW4gZGlhcml6YXRpb24gc2VnbWVudDoKICAgICAgICBsYXN0X2Nob3NlbiA9IHNlbGYuX3NlZ21lbnRzW3NlbGYuX2xhc3RfY2hvc2VuX2luZGV4XQoKICAgICAgICAjIE5vbmUgdmFsdWUgbWF5IGFwcGVhciBpZiB0aGUgd29yZCBpcyB0aGUgbGFzdCB3b3JkIGluIHRoZSBhdWRpbyBmaWxlLCBvciBpdCB3YXMgc3BsaXQgZHVyaW5nIGluZmVyZW5jZS4gSW4KICAgICAgICAjIHRoYXQgY2FzZSwgd2UnbGwgc2V0IHRoZSBsYXN0IHNlZ21lbnQ6CiAgICAgICAgaWYgd29yZC5lbmQgaXMgTm9uZToKICAgICAgICAgICAgc2VsZi5fbGFzdF9jaG9zZW5faW5kZXggPSBsZW4oc2VsZi5fc2VnbWVudHMpIC0gMQogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyBJZiB0aGUgd29yZCBlbmRzIGJlZm9yZSB0aGUgbGFzdCBjaG9zZW4gc2VnbWVudDoKICAgICAgICBpZiB3b3JkLmVuZCA8PSBsYXN0X2Nob3Nlbi5zdGFydDoKICAgICAgICAgICAgIyBUaGVuIGl0IGlzIHN0aWxsIHRoZSBjbG9zZXN0IHNlZ21lbnQKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgICMgV2UgY2hlY2sgaWYgaXQgZW5kcyBpbnNpZGUgdGhlIGxhc3QgY2hvc2VuIHNlZ21lbnQ6CiAgICAgICAgaWYgd29yZC5lbmQgPCBsYXN0X2Nob3Nlbi5lbmQ6CiAgICAgICAgICAgICMgVGhlbiBpdCBzdGlsbCBpcyB0aGUgY2xvc2VzdCBzZWdtZW50CiAgICAgICAgICAgIHJldHVybgoKICAgICAgICAjIFRoZSB3b3JkIGVuZHMgYWZ0ZXIgdGhlIHNlZ21lbnQsIHdlIG5lZWQgdG8gY29sbGVjdCBhbGwgbmV4dCBzZWdtZW50cyB1cCB1bnRpbCB0aGUgd29yZCBlbmRzIGJlZm9yZSB0aGVtOgogICAgICAgIHBvc3NpYmxlX3NlZ21lbnRzID0gW3NlbGYuX2xhc3RfY2hvc2VuX2luZGV4XQogICAgICAgIGZvciBpIGluIHJhbmdlKHNlbGYuX2xhc3RfY2hvc2VuX2luZGV4ICsgMSwgbGVuKHNlbGYuX3NlZ21lbnRzKSk6CiAgICAgICAgICAgIGlmIHdvcmQuZW5kID4gc2VsZi5fc2VnbWVudHNbaV0uZW5kOgogICAgICAgICAgICAgICAgcG9zc2libGVfc2VnbWVudHMuYXBwZW5kKGkpCiAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICBwb3NzaWJsZV9zZWdtZW50cy5hcHBlbmQoaSkKICAgICAgICAgICAgYnJlYWsKCiAgICAgICAgIyBDaGVjayBmb3IgdGhlIG1vc3Qgb3ZlcmxhcHBpbmcgb3B0aW9uOgogICAgICAgIGJlc3Rfb3ZlcmxhcCA9IDAKICAgICAgICBtb3N0X292ZXJsYXBwaW5nX3NlZ21lbnRfaW5kZXggPSBOb25lCiAgICAgICAgZm9yIGkgaW4gcG9zc2libGVfc2VnbWVudHM6CiAgICAgICAgICAgICMgSWYgdGhlIHdvcmQgc3RhcnRzIGJlZm9yZSBzZWdtZW50OgogICAgICAgICAgICBpZiB3b3JkLnN0YXJ0IDw9IHNlbGYuX3NlZ21lbnRzW2ldLnN0YXJ0OgogICAgICAgICAgICAgICAgIyBJZiBpdCBlbmRzIGJlZm9yZSB0aGUgc2VnbWVudCwgdGhlcmUgaXMgYW4gb3ZlcmxhcCBmcm9tIHRoZSBzdGFydCBvZiB0aGUgc2VnbWVudCB0byB0aGUgZW5kIG9mIHRoZQogICAgICAgICAgICAgICAgIyB3b3JkOgogICAgICAgICAgICAgICAgaWYgd29yZC5lbmQgPCBzZWxmLl9zZWdtZW50c1tpXS5lbmQ6CiAgICAgICAgICAgICAgICAgICAgb3ZlcmxhcCA9IHdvcmQuZW5kIC0gc2VsZi5fc2VnbWVudHNbaV0uc3RhcnQKICAgICAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAgICAgIyBUaGUgd29yZCBpcyB3cmFwcGluZyB0aGUgc2VnbWVudCwgdGhlIG92ZXJsYXAgaXMgdGhlIHNlZ21lbnQncyBsZW5ndGg6CiAgICAgICAgICAgICAgICAgICAgb3ZlcmxhcCA9IHNlbGYuX3NlZ21lbnRzW2ldLmVuZCAtIHNlbGYuX3NlZ21lbnRzW2ldLnN0YXJ0CiAgICAgICAgICAgICMgVGhlIHdvcmQgc3RhcnRzIGluIHNlZ21lbnQsIGNoZWNrIGlmIHRoZSB3b3JkIGVuZHMgaW4gaXQ6CiAgICAgICAgICAgIGVsaWYgd29yZC5lbmQgPCBzZWxmLl9zZWdtZW50c1tpXS5lbmQ6CiAgICAgICAgICAgICAgICAjIFRoZSBvdmVybGFwIGlzIHRoZSB3b3JkJ3MgbGVuZ3RoOgogICAgICAgICAgICAgICAgb3ZlcmxhcCA9IHdvcmQuZW5kIC0gd29yZC5zdGFydAogICAgICAgICAgICAjIFRoZSB3b3JkIHN0YXJ0IGluIHNlZ21lbnQgYnV0IGVuZHMgYWZ0ZXIgaXQsIHRoZSBvdmVybGFwIGlzIGZyb20gdGhlIHdvcmQncyBzdGFydCB0byB0aGUgc2VnbWVudCdzIGVuZDoKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIG92ZXJsYXAgPSBzZWxmLl9zZWdtZW50c1tpXS5lbmQgLSB3b3JkLnN0YXJ0CiAgICAgICAgICAgICMgQ2hlY2sgZm9yIG5ldyBiZXN0IG92ZXJsYXA6CiAgICAgICAgICAgIGlmIG92ZXJsYXAgPiBiZXN0X292ZXJsYXA6CiAgICAgICAgICAgICAgICBiZXN0X292ZXJsYXAgPSBvdmVybGFwCiAgICAgICAgICAgICAgICBtb3N0X292ZXJsYXBwaW5nX3NlZ21lbnRfaW5kZXggPSBpCiAgICAgICAgaWYgbW9zdF9vdmVybGFwcGluZ19zZWdtZW50X2luZGV4IGlzIG5vdCBOb25lOgogICAgICAgICAgICBzZWxmLl9sYXN0X2Nob3Nlbl9pbmRleCA9IG1vc3Rfb3ZlcmxhcHBpbmdfc2VnbWVudF9pbmRleAogICAgICAgICAgICByZXR1cm4KCiAgICAgICAgIyBJZiB0aGVyZSBpcyBubyBvdmVybGFwcGluZyBzZWdtZW50LCByZXR1cm4gdGhlIGNsb3Nlc3Qgc2VnbWVudDoKICAgICAgICBiZXN0X2Rpc3RhbmNlID0gTm9uZQogICAgICAgIGNsb3Nlc3Rfc2VnbWVudF9pbmRleCA9IE5vbmUKICAgICAgICBmb3IgaSBpbiBwb3NzaWJsZV9zZWdtZW50czoKICAgICAgICAgICAgZGlzdGFuY2UgPSAoCiAgICAgICAgICAgICAgICB3b3JkLnN0YXJ0IC0gc2VsZi5fc2VnbWVudHNbaV0uZW5kCiAgICAgICAgICAgICAgICBpZiB3b3JkLnN0YXJ0ID4gc2VsZi5fc2VnbWVudHNbaV0uZW5kCiAgICAgICAgICAgICAgICBlbHNlIHNlbGYuX3NlZ21lbnRzW2ldLnN0YXJ0IC0gd29yZC5lbmQKICAgICAgICAgICAgKQogICAgICAgICAgICBpZiBiZXN0X2Rpc3RhbmNlIGlzIE5vbmUgb3IgZGlzdGFuY2UgPCBiZXN0X2Rpc3RhbmNlOgogICAgICAgICAgICAgICAgYmVzdF9kaXN0YW5jZSA9IGRpc3RhbmNlCiAgICAgICAgICAgICAgICBjbG9zZXN0X3NlZ21lbnRfaW5kZXggPSBpCiAgICAgICAgc2VsZi5fbGFzdF9jaG9zZW5faW5kZXggPSBjbG9zZXN0X3NlZ21lbnRfaW5kZXgKCgpjbGFzcyBTcGVlY2hEaWFyaXphdGlvblBlckNoYW5uZWxUYXNrKEJhc2VUYXNrKToKICAgICIiIgogICAgQSB0YXNrIHRvIHdyaXRlIHRoZSB0cmFuc2NyaXB0aW9uIHRvIGZpbGUgd2l0aCByZXNwZWN0IHRvIGEgZ2l2ZW4gc3BlZWNoIGRpYXJpemF0aW9uIHBlciBjaGFubmVsLgogICAgIiIiCgogICAgY2xhc3MgX1dvcmRUaW1lc3RhbXAoTmFtZWRUdXBsZSk6CiAgICAgICAgIiIiCiAgICAgICAgQSB3b3JkIHdpdGggaXRzIHN0YXJ0IGFuZCBlbmQgdGltZXN0YW1wcyBhbmQgc3BlYWtlciBsYWJlbCAoY2hhbm5lbCB0aGUgd29yZCB3YXMgdGFrZW4gZnJvbSkuCiAgICAgICAgIiIiCgogICAgICAgIHN0YXJ0OiBmbG9hdAogICAgICAgIGVuZDogZmxvYXQKICAgICAgICBzcGVha2VyOiBzdHIKICAgICAgICB0ZXh0OiBzdHIKCiAgICBkZWYgX19pbml0X18oc2VsZiwgYXVkaW9fZmlsZTogUGF0aCwgdGV4dF9maWxlOiBQYXRoKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSB0YXNrLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogUGF0aCB0byB0aGUgYXVkaW8gZmlsZSB0aGF0IHdhcyB0cmFuc2NyaWJlZC4KICAgICAgICA6cGFyYW0gdGV4dF9maWxlOiAgUGF0aCB0byB0aGUgdGV4dCBmaWxlIHRvIHdyaXRlIHRoZSB0cmFuc2NyaXB0aW9uIHRvLgogICAgICAgICIiIgogICAgICAgIHN1cGVyKCkuX19pbml0X18oCiAgICAgICAgICAgIGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSwgdHJhbnNjcmlwdGlvbl9vdXRwdXQ9e30sIHRleHRfZmlsZT10ZXh0X2ZpbGUKICAgICAgICApCiAgICAgICAgc2VsZi5fdHJhbnNjcmlwdGlvbl9vdXRwdXRfY2hhbm5lbHM6IExpc3RbVHVwbGVbc3RyLCBkaWN0XV0gPSBbXQoKICAgIEBwcm9wZXJ0eQogICAgZGVmIHRyYW5zY3JpcHRpb25fb3V0cHV0X2NoYW5uZWxzKHNlbGYpIC0+IExpc3RbVHVwbGVbc3RyLCBkaWN0XV06CiAgICAgICAgIiIiCiAgICAgICAgR2V0IHRoZSB0cmFuc2NyaXB0aW9uIG91dHB1dCBjaGFubmVscy4KCiAgICAgICAgOnJldHVybnM6IFRoZSB0cmFuc2NyaXB0aW9uIG91dHB1dCBjaGFubmVscy4KICAgICAgICAiIiIKICAgICAgICByZXR1cm4gc2VsZi5fdHJhbnNjcmlwdGlvbl9vdXRwdXRfY2hhbm5lbHMKCiAgICBkZWYgZG9fdGFzayhzZWxmKToKICAgICAgICAiIiIKICAgICAgICBUcnkgdG8gcGVyZm9ybSB0aGUgdGFzayBzdG9yaW5nIGFuIGVycm9yIGlmIG9jY3VycmVkLgogICAgICAgICIiIgogICAgICAgIGZvciBfLCBjaGFubmVsX291dHB1dCBpbiBzZWxmLl90cmFuc2NyaXB0aW9uX291dHB1dF9jaGFubmVsczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShjaGFubmVsX291dHB1dCwgc3RyKToKICAgICAgICAgICAgICAgIHNlbGYuX2Vycm9yID0gc2VsZi5fdHJhbnNjcmlwdGlvbl9vdXRwdXRfY2hhbm5lbHMKICAgICAgICAgICAgICAgIHJldHVybgogICAgICAgIHN1cGVyKCkuZG9fdGFzaygpCgogICAgZGVmIHRvX3R1cGxlKHNlbGYpIC0+IFR1cGxlW3N0ciwgZGljdF06CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCB0aGUgdGFzayB0byBhIHR1cGxlIHRvIHJlY29uc3RydWN0IGl0IGxhdGVyICh1c2VkIGZvciBtdWx0aXByb2Nlc3NpbmcgdG8gcGFzcyBpbiBxdWV1ZSkuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgY29udmVydGVkIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgdGFza19jbGFzcywgdGFza19rd2FyZ3MgPSBzdXBlcigpLnRvX3R1cGxlKCkKICAgICAgICB0YXNrX2t3YXJncy5wb3AoInRyYW5zY3JpcHRpb25fb3V0cHV0IikKICAgICAgICByZXR1cm4gdGFza19jbGFzcywgdGFza19rd2FyZ3MKCiAgICBkZWYgX2RvX3Rhc2soc2VsZik6CiAgICAgICAgIiIiCiAgICAgICAgUGVyZm9ybSB0aGUgdGFzayAtIHdyaXRlIHRoZSB0cmFuc2NyaXB0aW9uIHRvIHRoZSBzdG9yZWQgZmlsZSBwYXRoIHdpdGggcmVzcGVjdCB0byB0aGUgZ2l2ZW4gc3BlZWNoIGRpYXJpemF0aW9uCiAgICAgICAgcGVyIGNoYW5uZWwuCiAgICAgICAgIiIiCiAgICAgICAgIyBDYXN0IHRoZSBjaHVua3MgdG8gd29yZCB0aW1lc3RhbXBzIHR1cGxlczoKICAgICAgICB3b3Jkc19wZXJfY2hhbm5lbCA9IFsKICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgU3BlZWNoRGlhcml6YXRpb25QZXJDaGFubmVsVGFzay5fV29yZFRpbWVzdGFtcCgKICAgICAgICAgICAgICAgICAgICBzdGFydD1jaHVua1sidGltZXN0YW1wIl1bMF0sCiAgICAgICAgICAgICAgICAgICAgZW5kPWNodW5rWyJ0aW1lc3RhbXAiXVsxXSwKICAgICAgICAgICAgICAgICAgICBzcGVha2VyPXNwZWFrZXIsCiAgICAgICAgICAgICAgICAgICAgdGV4dD1jaHVua1sidGV4dCJdLAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZm9yIGNodW5rIGluIG91dHB1dFsiY2h1bmtzIl0KICAgICAgICAgICAgXQogICAgICAgICAgICBmb3Igc3BlYWtlciwgb3V0cHV0IGluIHNlbGYuX3RyYW5zY3JpcHRpb25fb3V0cHV0X2NoYW5uZWxzCiAgICAgICAgXQoKICAgICAgICAjIE1lcmdlIGFuZCBzb3J0IHRoZSB3b3JkcyBwZXIgY2hhbm5lbCBieSB0aGVpciBzdGFydCB0aW1lOgogICAgICAgIHdvcmRzID0gb3BlcmF0b3IuYWRkKCp3b3Jkc19wZXJfY2hhbm5lbCkKICAgICAgICB3b3Jkcy5zb3J0KCkKCiAgICAgICAgIyBXcml0ZSB0aGUgdHJhbnNjcmlwdGlvbiB0byBmaWxlOgogICAgICAgIGN1cnJlbnRfc3BlYWtlciA9IHdvcmRzWzBdLnNwZWFrZXIKICAgICAgICB0ZXh0ID0gZiJ7Y3VycmVudF9zcGVha2VyfToiCiAgICAgICAgZm9yIHdvcmQgaW4gd29yZHM6CiAgICAgICAgICAgICMgQ2hlY2sgaWYgdGhlIHdvcmQncyBzcGVha2VyIGlzIGRpZmZlcmVudCBmcm9tIHRoZSBjdXJyZW50IG9uZToKICAgICAgICAgICAgaWYgd29yZC5zcGVha2VyICE9IGN1cnJlbnRfc3BlYWtlcjoKICAgICAgICAgICAgICAgICMgQXBwZW5kIGEgbmV3bGluZSBhbmQgdXBkYXRlIHRoZSBuZXcgc3BlYWtlcjoKICAgICAgICAgICAgICAgIGN1cnJlbnRfc3BlYWtlciA9IHdvcmQuc3BlYWtlcgogICAgICAgICAgICAgICAgdGV4dCArPSBmIlxue2N1cnJlbnRfc3BlYWtlcn06IgogICAgICAgICAgICAjIENvbGxlY3QgdGhlIHdvcmQ6CiAgICAgICAgICAgIHRleHQgKz0gd29yZC50ZXh0CgogICAgICAgICMgVXBkYXRlIHRoZSB0cmFuc2NyaXB0aW9uIG91dHB1dCB3aXRoIHRoZSBuZXcgdGV4dCB0byB3cml0ZSBpdCB0byBmaWxlOgogICAgICAgIHNlbGYuX3RyYW5zY3JpcHRpb25fb3V0cHV0WyJ0ZXh0Il0gPSB0ZXh0CiAgICAgICAgc3VwZXIoKS5fZG9fdGFzaygpCgoKY2xhc3MgQmF0Y2hQcm9jZXNzb3I6CiAgICAiIiIKICAgIEEgYmF0Y2ggcHJvY2Vzc29yIHRvIHByb2Nlc3MgYmF0Y2hlcyBvZiB0cmFuc2NyaXB0aW9ucy4gVGhlIGJhdGNoIHByb2Nlc3NvciBpcyBjcmVhdGluZyB0YXNrcyBhbmQgaXMgYWltZWQgdG8gYmUKICAgIHdvcmtpbmcgYWxvbmcgdGhlIHRyYW5zY3JpYmVyLiBJdCBjYW4gYmUgdXNlZCB3aXRoIG11bHRpcHJvY2Vzc2luZyBxdWV1ZSBvciBydW4gdGhlIHRhc2tzIGRpcmVjdGx5IHVzaW5nIHRoZQogICAgYXNzb2NpYXRlZCBtZXRob2RzLgogICAgIiIiCgogICAgZGVmIF9faW5pdF9fKHNlbGYsIGF1ZGlvX2ZpbGVzOiBMaXN0W1BhdGhdLCBvdXRwdXRfZGlyZWN0b3J5OiBQYXRoKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSBiYXRjaCBwcm9jZXNzb3IuCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlczogICAgICBUaGUgbGlzdCBvZiBhbGwgYXVkaW8gZmlsZXMgdG8gdHJhbnNjcmliZS4KICAgICAgICA6cGFyYW0gb3V0cHV0X2RpcmVjdG9yeTogVGhlIG91dHB1dCBkaXJlY3RvcnkgdG8gd3JpdGUgdGhlIHRyYW5zY3JpcHRpb25zIHRvLgogICAgICAgICIiIgogICAgICAgICMgU3RvcmUgdGhlIHBhcmFtZXRlcnM6CiAgICAgICAgc2VsZi5fYXVkaW9fZmlsZXMgPSBhdWRpb19maWxlcwogICAgICAgIHNlbGYuX291dHB1dF9kaXJlY3RvcnkgPSBvdXRwdXRfZGlyZWN0b3J5CgogICAgICAgICMgUHJlcGFyZSB0aGUgYmF0Y2hpbmcgdmFyaWFibGVzOgogICAgICAgIHNlbGYuX2N1cnJlbnRfZmlsZV9pbmRleCA9IDAKICAgICAgICBzZWxmLl90YXNrczogTGlzdFtCYXNlVGFza10gPSBbXQogICAgICAgIHNlbGYuX3Jlc3VsdHM6IExpc3RbVHVwbGVbYm9vbCwgVHVwbGVbc3RyLCBzdHJdXV0gPSBbXQoKICAgIGRlZiBwcm9jZXNzX2JhdGNoKHNlbGYsIGJhdGNoOiBMaXN0W1VuaW9uW2RpY3QsIHN0cl1dKToKICAgICAgICAiIiIKICAgICAgICBQcm9jZXNzIGEgYmF0Y2ggb2YgdHJhbnNjcmlwdGlvbnMuIFRhc2tzIHJlbGF0ZWQgdG8gdGhlIGdpdmVuIGJhdGNoIHdpbGwgYmUgY3JlYXRlZCBhbmQgc3RvcmVkIGluIHRoZSBiYXRjaAogICAgICAgIHByb2Nlc3Nvci4KCiAgICAgICAgOnBhcmFtIGJhdGNoOiBUaGUgYmF0Y2ggb2YgdHJhbnNjcmlwdGlvbnMgdG8gcHJvY2Vzcy4KICAgICAgICAiIiIKICAgICAgICAjIEdldCB0aGUgcmVsZXZhbnQgZmlsZXMgYmVsb25ncyB0byB0aGUgZ2l2ZW4gYmF0Y2g6CiAgICAgICAgY3VycmVudF9maWxlcyA9IHNlbGYuX2dldF9jdXJyZW50X2ZpbGVzKGJhdGNoX3NpemU9bGVuKGJhdGNoKSkKCiAgICAgICAgIyBCdWlsZCB0aGUgZGlhcml6YXRpb24gdGFza3M6CiAgICAgICAgc2VsZi5fdGFza3MuZXh0ZW5kKAogICAgICAgICAgICBbCiAgICAgICAgICAgICAgICBCYXNlVGFzaygKICAgICAgICAgICAgICAgICAgICBhdWRpb19maWxlPWZpbGUsCiAgICAgICAgICAgICAgICAgICAgdHJhbnNjcmlwdGlvbl9vdXRwdXQ9YmF0Y2hbaV0sCiAgICAgICAgICAgICAgICAgICAgdGV4dF9maWxlPXNlbGYuX291dHB1dF9kaXJlY3RvcnkgLyBmIntmaWxlLnN0ZW19LnR4dCIsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBmb3IgaSwgZmlsZSBpbiBlbnVtZXJhdGUoY3VycmVudF9maWxlcykKICAgICAgICAgICAgXQogICAgICAgICkKCiAgICBkZWYgZ2V0X3Rhc2tzKHNlbGYpIC0+IExpc3RbQmFzZVRhc2tdOgogICAgICAgICIiIgogICAgICAgIEdldCB0aGUgdGFza3MgdG8gcGVyZm9ybS4KCiAgICAgICAgOnJldHVybnM6IFRoZSB0YXNrcyB0byBwZXJmb3JtLgogICAgICAgICIiIgogICAgICAgIHRhc2tzID0gc2VsZi5fdGFza3MKICAgICAgICBzZWxmLl90YXNrcyA9IFtdCiAgICAgICAgcmV0dXJuIHRhc2tzCgogICAgZGVmIGRvX3Rhc2tzKHNlbGYpOgogICAgICAgICIiIgogICAgICAgIFBlcmZvcm0gdGhlIHRhc2tzLiBTaG91bGQgYmUgdXNlZCBpZiBubyBtdWx0aXByb2Nlc3NpbmcgcXVldWUgaXMgZ2l2ZW4gdG8gYSB0cmFuc2NyaWJlci4KICAgICAgICAiIiIKICAgICAgICBmb3IgdGFzayBpbiBzZWxmLmdldF90YXNrcygpOgogICAgICAgICAgICB0YXNrLmRvX3Rhc2soKQogICAgICAgICAgICBzZWxmLl9yZXN1bHRzLmFwcGVuZCgodGFzay5pc19mYWlsZWQoKSwgdGFzay5nZXRfcmVzdWx0KCkpKQoKICAgIGRlZiBnZXRfcmVzdWx0cyhzZWxmKSAtPiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgc3RyXV1dOgogICAgICAgICIiIgogICAgICAgIEdldCB0aGUgcmVzdWx0cyBvZiB0aGUgdGFza3MuIFRoZSBzdG9yZWQgcmVzdWx0cyBhcmUgdGhlbiBjbGVhcmVkLgoKICAgICAgICA6cmV0dXJuczogVGhlIHJlc3VsdHMgb2YgdGhlIHRhc2tzLgogICAgICAgICIiIgogICAgICAgIHJlc3VsdHMgPSBzZWxmLl9yZXN1bHRzCiAgICAgICAgc2VsZi5fcmVzdWx0cyA9IFtdCiAgICAgICAgcmV0dXJuIHJlc3VsdHMKCiAgICBkZWYgX2dldF9jdXJyZW50X2ZpbGVzKHNlbGYsIGJhdGNoX3NpemU6IGludCkgLT4gTGlzdFtQYXRoXToKICAgICAgICAiIiIKICAgICAgICBHZXQgdGhlIGN1cnJlbnQgZmlsZXMgdG8gcHJvY2Vzcy4KCiAgICAgICAgOnBhcmFtIGJhdGNoX3NpemU6IFRoZSBiYXRjaCBzaXplIHRvIHByb2dyZXNzIHRoZSBjdXJyZW50IGZpbGUgaW5kZXguCgogICAgICAgIDpyZXR1cm5zOiBUaGUgY3VycmVudCBmaWxlcyB0byBwcm9jZXNzLgogICAgICAgICIiIgogICAgICAgIGVuZF9pbmRleCA9ICgKICAgICAgICAgICAgc2VsZi5fY3VycmVudF9maWxlX2luZGV4ICsgYmF0Y2hfc2l6ZQogICAgICAgICAgICBpZiBzZWxmLl9jdXJyZW50X2ZpbGVfaW5kZXggKyBiYXRjaF9zaXplIDwgbGVuKHNlbGYuX2F1ZGlvX2ZpbGVzKQogICAgICAgICAgICBlbHNlIGxlbihzZWxmLl9hdWRpb19maWxlcykKICAgICAgICApCiAgICAgICAgY3VycmVudF9maWxlcyA9IHNlbGYuX2F1ZGlvX2ZpbGVzW3NlbGYuX2N1cnJlbnRfZmlsZV9pbmRleCA6IGVuZF9pbmRleF0KICAgICAgICBzZWxmLl9jdXJyZW50X2ZpbGVfaW5kZXggPSBlbmRfaW5kZXgKICAgICAgICByZXR1cm4gY3VycmVudF9maWxlcwoKCmNsYXNzIFNwZWVjaERpYXJpemF0aW9uQmF0Y2hQcm9jZXNzb3IoQmF0Y2hQcm9jZXNzb3IpOgogICAgIiIiCiAgICBBIGJhdGNoIHByb2Nlc3NvciB0byBwcm9jZXNzIGJhdGNoZXMgb2YgdHJhbnNjcmlwdGlvbnMgd2l0aCByZXNwZWN0IHRvIGEgZ2l2ZW4gc3BlZWNoIGRpYXJpemF0aW9uLiBUaGUgYmF0Y2gKICAgIHByb2Nlc3NvciBpcyBjcmVhdGluZyB0YXNrcyBhbmQgaXMgYWltZWQgdG8gYmUgd29ya2luZyBhbG9uZyB0aGUgdHJhbnNjcmliZXIuIEl0IGNhbiBiZSB1c2VkIHdpdGggbXVsdGlwcm9jZXNzaW5nCiAgICBxdWV1ZSBvciBydW4gdGhlIHRhc2tzIGRpcmVjdGx5IHVzaW5nIHRoZSBhc3NvY2lhdGVkIG1ldGhvZHMuCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwgYXVkaW9fZmlsZXM6IExpc3RbUGF0aF0sIG91dHB1dF9kaXJlY3Rvcnk6IFBhdGgsIHNwZWVjaF9kaWFyaXphdGlvbjogZGljdAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSBiYXRjaCBwcm9jZXNzb3IuCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlczogICAgICAgIFRoZSBsaXN0IG9mIGFsbCBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlLgogICAgICAgIDpwYXJhbSBvdXRwdXRfZGlyZWN0b3J5OiAgIFRoZSBvdXRwdXQgZGlyZWN0b3J5IHRvIHdyaXRlIHRoZSB0cmFuc2NyaXB0aW9ucyB0by4KICAgICAgICA6cGFyYW0gc3BlZWNoX2RpYXJpemF0aW9uOiBBIHNwZWVjaCBkaWFyaXphdGlvbiBkaWN0aW9uYXJ5IHRvIHBhc3MgYWxvbmcgd2l0aCBlYWNoIHByb2Nlc3NlZCBiYXRjaC4KICAgICAgICAiIiIKICAgICAgICBzdXBlcigpLl9faW5pdF9fKGF1ZGlvX2ZpbGVzPWF1ZGlvX2ZpbGVzLCBvdXRwdXRfZGlyZWN0b3J5PW91dHB1dF9kaXJlY3RvcnkpCiAgICAgICAgc2VsZi5fc3BlZWNoX2RpYXJpemF0aW9uID0gc3BlZWNoX2RpYXJpemF0aW9uCiAgICAgICAgc2VsZi5fYXVkaW9fZmlsZXMgPSBhdWRpb19maWxlcwoKICAgIGRlZiBwcm9jZXNzX2JhdGNoKHNlbGYsIGJhdGNoOiBMaXN0W2RpY3RdKToKICAgICAgICAiIiIKICAgICAgICBQcm9jZXNzIGEgYmF0Y2ggb2YgdHJhbnNjcmlwdGlvbnMuIFRhc2tzIHJlbGF0ZWQgdG8gdGhlIGdpdmVuIGJhdGNoIHdpbGwgYmUgY3JlYXRlZCBhbmQgc3RvcmVkIGluIHRoZSBiYXRjaAogICAgICAgIHByb2Nlc3Nvci4KCiAgICAgICAgOnBhcmFtIGJhdGNoOiBUaGUgYmF0Y2ggb2YgdHJhbnNjcmlwdGlvbnMgdG8gcHJvY2Vzcy4KICAgICAgICAiIiIKICAgICAgICAjIEdldCB0aGUgcmVsZXZhbnQgZmlsZXMgYmVsb25ncyB0byB0aGUgZ2l2ZW4gYmF0Y2g6CiAgICAgICAgY3VycmVudF9maWxlcyA9IHNlbGYuX2dldF9jdXJyZW50X2ZpbGVzKGJhdGNoX3NpemU9bGVuKGJhdGNoKSkKCiAgICAgICAgIyBCdWlsZCB0aGUgZGlhcml6YXRpb24gdGFza3M6CiAgICAgICAgc2VsZi5fdGFza3MuZXh0ZW5kKAogICAgICAgICAgICBbCiAgICAgICAgICAgICAgICBTcGVlY2hEaWFyaXphdGlvblRhc2soCiAgICAgICAgICAgICAgICAgICAgYXVkaW9fZmlsZT1maWxlLAogICAgICAgICAgICAgICAgICAgIHRyYW5zY3JpcHRpb25fb3V0cHV0PWJhdGNoW2ldLAogICAgICAgICAgICAgICAgICAgIHRleHRfZmlsZT1zZWxmLl9vdXRwdXRfZGlyZWN0b3J5IC8gZiJ7ZmlsZS5zdGVtfS50eHQiLAogICAgICAgICAgICAgICAgICAgIHNwZWVjaF9kaWFyaXphdGlvbj1zZWxmLl9zcGVlY2hfZGlhcml6YXRpb24uZ2V0KGZpbGUubmFtZSksCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBmb3IgaSwgZmlsZSBpbiBlbnVtZXJhdGUoY3VycmVudF9maWxlcykKICAgICAgICAgICAgXQogICAgICAgICkKCgpjbGFzcyBQZXJDaGFubmVsU3BlZWNoRGlhcml6YXRpb25CYXRjaFByb2Nlc3NvcihCYXRjaFByb2Nlc3Nvcik6CiAgICAiIiIKICAgIEEgYmF0Y2ggcHJvY2Vzc29yIHRvIHByb2Nlc3MgYmF0Y2hlcyBvZiB0cmFuc2NyaXB0aW9ucyBwZXIgY2hhbm5lbC4gVGhlIGJhdGNoIHByb2Nlc3NvciBpcyBjcmVhdGluZyB0YXNrcyB3aXRoIHRoZQogICAgc2VsZWN0ZWQgYW1vdW50IG9mIGNoYW5uZWxzIGdpdmVuIGFuZCBpcyBhaW1lZCB0byBiZSB3b3JraW5nIGFsb25nIHRoZSB0cmFuc2NyaWJlci4gSXQgY2FuIGJlIHVzZWQgd2l0aAogICAgbXVsdGlwcm9jZXNzaW5nIHF1ZXVlIG9yIHJ1biB0aGUgdGFza3MgZGlyZWN0bHkgdXNpbmcgdGhlIGFzc29jaWF0ZWQgbWV0aG9kcy4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgIGF1ZGlvX2ZpbGVzOiBMaXN0W1BhdGhdLAogICAgICAgIG91dHB1dF9kaXJlY3Rvcnk6IFBhdGgsCiAgICAgICAgbl9jaGFubmVsczogaW50LAogICAgICAgIHNwZWFrZXJzOiBMaXN0W3N0cl0sCiAgICApOgogICAgICAgICIiIgogICAgICAgIEluaXRpYWxpemUgdGhlIGJhdGNoIHByb2Nlc3Nvci4KCiAgICAgICAgOnBhcmFtIGF1ZGlvX2ZpbGVzOiAgICAgIFRoZSBsaXN0IG9mIGFsbCBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlLgogICAgICAgIDpwYXJhbSBvdXRwdXRfZGlyZWN0b3J5OiBUaGUgb3V0cHV0IGRpcmVjdG9yeSB0byB3cml0ZSB0aGUgdHJhbnNjcmlwdGlvbnMgdG8uCiAgICAgICAgOnBhcmFtIG5fY2hhbm5lbHM6ICAgICAgIFRoZSBudW1iZXIgb2YgY2hhbm5lbHMgaW4gZWFjaCBhdWRpbyBmaWxlIHRvIHRyYW5zY3JpYmUuCiAgICAgICAgOnBhcmFtIHNwZWFrZXJzOiAgICAgICAgIFRoZSBzcGVha2VycyBsYWJlbHMgdG8gdXNlIGZvciBlYWNoIGNoYW5uZWwuCiAgICAgICAgIiIiCiAgICAgICAgc3VwZXIoKS5fX2luaXRfXyhhdWRpb19maWxlcz1hdWRpb19maWxlcywgb3V0cHV0X2RpcmVjdG9yeT1vdXRwdXRfZGlyZWN0b3J5KQoKICAgICAgICAjIFN0b3JlIHRoZSBwYXJhbWV0ZXJzOgogICAgICAgIHNlbGYuX25fY2hhbm5lbHMgPSBuX2NoYW5uZWxzCiAgICAgICAgc2VsZi5fc3BlYWtlcnMgPSBzcGVha2VycwoKICAgICAgICAjIFByZXBhcmUgYSBjaGFubmVsIGJ1ZmZlciB0byBzdG9yZSB0aGUgY2hhbm5lbHMgdW50aWwgdGhlIGN1cnJlbnQgdGFzayBjcmVhdGVkIGlzIGZ1bGx5IGNvdmVyZWQ6CiAgICAgICAgc2VsZi5fdGFza19pbl9wcm9jZXNzOiBTcGVlY2hEaWFyaXphdGlvblBlckNoYW5uZWxUYXNrID0gTm9uZQoKICAgIGRlZiBwcm9jZXNzX2JhdGNoKHNlbGYsIGJhdGNoOiBMaXN0W2RpY3RdKToKICAgICAgICAiIiIKICAgICAgICBQcm9jZXNzIGEgYmF0Y2ggb2YgdHJhbnNjcmlwdGlvbnMuIFRhc2tzIHJlbGF0ZWQgdG8gdGhlIGdpdmVuIGJhdGNoIHdpbGwgYmUgY3JlYXRlZCBhbmQgc3RvcmVkIGluIHRoZSBiYXRjaAogICAgICAgIHByb2Nlc3Nvci4KCiAgICAgICAgOnBhcmFtIGJhdGNoOiBUaGUgYmF0Y2ggb2YgdHJhbnNjcmlwdGlvbnMgdG8gcHJvY2Vzcy4KICAgICAgICAiIiIKICAgICAgICAjIEdvIG92ZXIgdGhlIGJhdGNoIGFuZCBjcmVhdGUgdGhlIHRhc2tzOgogICAgICAgIGZvciBvdXRwdXQgaW4gYmF0Y2g6CiAgICAgICAgICAgICMgQ2hlY2sgaWYgdGhlcmUgaXMgYSB0YXNrIGluIHByb2Nlc3M6CiAgICAgICAgICAgIGlmIG5vdCBzZWxmLl90YXNrX2luX3Byb2Nlc3M6CiAgICAgICAgICAgICAgICAjIENyZWF0ZSBhIG5ldyB0YXNrOgogICAgICAgICAgICAgICAgc2VsZi5fdGFza19pbl9wcm9jZXNzID0gU3BlZWNoRGlhcml6YXRpb25QZXJDaGFubmVsVGFzaygKICAgICAgICAgICAgICAgICAgICBhdWRpb19maWxlPXNlbGYuX2F1ZGlvX2ZpbGVzW3NlbGYuX2N1cnJlbnRfZmlsZV9pbmRleF0sCiAgICAgICAgICAgICAgICAgICAgdGV4dF9maWxlPXNlbGYuX291dHB1dF9kaXJlY3RvcnkKICAgICAgICAgICAgICAgICAgICAvIGYie3NlbGYuX2F1ZGlvX2ZpbGVzW3NlbGYuX2N1cnJlbnRfZmlsZV9pbmRleF0uc3RlbX0udHh0IiwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgIyBHZXQgdGhlIGNoYW5uZWwncyBzcGVha2VyOgogICAgICAgICAgICBzcGVha2VyID0gc2VsZi5fc3BlYWtlcnNbCiAgICAgICAgICAgICAgICBsZW4oc2VsZi5fdGFza19pbl9wcm9jZXNzLnRyYW5zY3JpcHRpb25fb3V0cHV0X2NoYW5uZWxzKQogICAgICAgICAgICBdCiAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgY2hhbm5lbCBpbnRvIHRoZSBwcm9jZXNzZWQgdGFzazoKICAgICAgICAgICAgc2VsZi5fdGFza19pbl9wcm9jZXNzLnRyYW5zY3JpcHRpb25fb3V0cHV0X2NoYW5uZWxzLmFwcGVuZCgKICAgICAgICAgICAgICAgIChzcGVha2VyLCBvdXRwdXQpCiAgICAgICAgICAgICkKICAgICAgICAgICAgIyBDaGVjayBpZiB0aGUgdGFzayBpcyBmdWxseSBjb3ZlcmVkIChhbGwgY2hhbm5lbHMgYXJlIGNvbGxlY3RlZCk6CiAgICAgICAgICAgIGlmICgKICAgICAgICAgICAgICAgIGxlbihzZWxmLl90YXNrX2luX3Byb2Nlc3MudHJhbnNjcmlwdGlvbl9vdXRwdXRfY2hhbm5lbHMpCiAgICAgICAgICAgICAgICA9PSBzZWxmLl9uX2NoYW5uZWxzCiAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICAjIENvbGxlY3QgdGhlIHRhc2sgYW5kIHJlc2V0IHRoZSB0YXNrIGluIHByb2Nlc3M6CiAgICAgICAgICAgICAgICBzZWxmLl90YXNrcy5hcHBlbmQoc2VsZi5fdGFza19pbl9wcm9jZXNzKQogICAgICAgICAgICAgICAgc2VsZi5fY3VycmVudF9maWxlX2luZGV4ICs9IDEKICAgICAgICAgICAgICAgIHNlbGYuX3Rhc2tfaW5fcHJvY2VzcyA9IE5vbmUKCgpjbGFzcyBUcmFuc2NyaWJlcjoKICAgICIiIgogICAgQSB0cmFuc2NyaXB0aW9uIHdyYXBwZXIgZm9yIHRoZSBIdWdnaW5nZmFjZSdzIEFTUiBwaXBlbGluZSAtCiAgICBodHRwczovL2h1Z2dpbmdmYWNlLmNvL3RyYW5zZm9ybWVycy9tYWluX2NsYXNzZXMvcGlwZWxpbmVzLmh0bWwjdHJhbnNmb3JtZXJzLkF1dG9tYXRpY1NwZWVjaFJlY29nbml0aW9uUGlwZWxpbmUgdG8KICAgIHVzZSB3aXRoIE9wZW5BSSdzIFdoaXNwZXIgbW9kZWxzIC0gaHR0cHM6Ly9odWdnaW5nZmFjZS5jby9vcGVuYWkuCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwKICAgICAgICBtb2RlbF9uYW1lOiBzdHIsCiAgICAgICAgZGV2aWNlOiBzdHIgPSBOb25lLAogICAgICAgIHVzZV9mbGFzaF9hdHRlbnRpb25fMjogYm9vbCA9IE5vbmUsCiAgICAgICAgdXNlX2JldHRlcl90cmFuc2Zvcm1lcnM6IGJvb2wgPSBOb25lLAogICAgICAgIGFzc2lzdGFudF9tb2RlbDogc3RyID0gTm9uZSwKICAgICAgICBtYXhfbmV3X3Rva2VuczogaW50ID0gMTI4LAogICAgICAgIGNodW5rX2xlbmd0aF9zOiBpbnQgPSAzMCwKICAgICAgICBiYXRjaF9zaXplOiBpbnQgPSAyLAogICAgICAgIHNwb2tlbl9sYW5ndWFnZTogc3RyID0gTm9uZSwKICAgICAgICB0cmFuc2xhdGVfdG9fZW5nbGlzaDogYm9vbCA9IEZhbHNlLAogICAgICAgIHJldHVybl90aW1lc3RhbXBzOiBVbmlvbltib29sLCBMaXRlcmFsWyJ3b3JkIl1dID0gRmFsc2UsCiAgICAgICAgcGVyX2NoYW5uZWxfdHJhbnNjcmlwdGlvbjogaW50ID0gMCwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgSW5pdGlhbGl6ZSB0aGUgdHJhbnNjcmliZXIuCgogICAgICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgICAgICAgICAgICBUaGUgbW9kZWwgbmFtZSB0byB1c2UuIFNob3VsZCBiZSBhIG1vZGVsIGZyb20gdGhlIE9wZW5BSSdzIFdoaXNwZXIgbW9kZWxzIGZvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZXN0IHJlc3VsdHMgKGZvciBleGFtcGxlICJ0aW55IiwgImJhc2UiLCAibGFyZ2UiLCBldGMuKS4KICAgICAgICA6cGFyYW0gZGV2aWNlOiAgICAgICAgICAgICAgICAgICAgVGhlIGRldmljZSB0byB1c2UgZm9yIGluZmVyZW5jZS4gSWYgbm90IGdpdmVuLCB3aWxsIHVzZSBHUFUgaWYgYXZhaWxhYmxlLgogICAgICAgIDpwYXJhbSB1c2VfZmxhc2hfYXR0ZW50aW9uXzI6ICAgICBXaGV0aGVyIHRvIHVzZSB0aGUgRmxhc2ggQXR0ZW50aW9uIDIgaW1wbGVtZW50YXRpb24uIEl0IGNhbiBiZSB1c2VkIG9ubHkgd2l0aAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvbmUgb2YgdGhlIGZvbGxvd2luZyBHUFVzOiBOdmlkaWEgSCBzZXJpZXMgYW5kIE52aWRpYSBBIHNlcmllcy4gVDQgc3VwcG9ydAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIGF2YWlsYWJsZSBzb29uLgoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90ZTogSWYgYm90aCBgdXNlX2ZsYXNoX2F0dGVudGlvbl8yYCBhbmQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYHVzZV9iZXR0ZXJfdHJhbnNmb3JtZXJzYCBhcmUgYE5vbmVgLCB0aGUgb3B0aW1pemF0aW9uIHdpbGwgYmUgY2hvc2VuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGF1dG9tYXRpY2FsbHkgYWNjb3JkaW5nIHRvIHRoZSBhdmFpbGFibGUgcmVzb3VyY2VzLgoKICAgICAgICA6cGFyYW0gdXNlX2JldHRlcl90cmFuc2Zvcm1lcnM6ICAgV2hldGhlciB0byB1c2UgdGhlIEJldHRlciBUcmFuc2Zvcm1lcnMgbGlicmFyeSB0byBmdXJ0aGVyIG9wdGltaXplIHRoZSBtb2RlbC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgU2hvdWxkIGJlIHVzZWQgZm9yIGFsbCB1c2UgY2FzZXMgdGhhdCBkbyBub3Qgc3VwcG9ydCBmbGFzaCBhdHRlbnRpb24gMi4KCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE5vdGU6IElmIGJvdGggYHVzZV9mbGFzaF9hdHRlbnRpb25fMmAgYW5kIGB1c2VfYmV0dGVyX3RyYW5zZm9ybWVyc2AgYXJlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBOb25lYCwgdGhlIG9wdGltaXphdGlvbiB3aWxsIGJlIGNob3NlbiBhdXRvbWF0aWNhbGx5IGFjY29yZGluZyB0byB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYXZhaWxhYmxlIHJlc291cmNlcy4KICAgICAgIDpwYXJhbSBhc3Npc3RhbnRfbW9kZWw6ICAgICAgICAgICBUaGUgYXNzaXN0YW50IG1vZGVsIG5hbWUgdG8gdXNlIGZvciBpbmZlcmVuY2UuIE5vdGljZSB0aGF0IHRoZSBvcHRpbWl6YXRpb25zCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChmbGFzaCBhdHRlbnRpb24gMiBhbmQgYmV0dGVyIHRyYW5zZm9ybWVycykgd2lsbCBiZSBhcHBsaWVkIGZvciB0aGUgYXNzaXN0YW50CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGFzIHdlbGwuIFNob3VsZCBiZSBhIG1vZGVsIGZyb20gSHVnZ2luZ2ZhY2UncyBkaXN0aWwtd2hpc3BlciAoc2VlIGhlcmUgZm9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vcmUgaW5mb3JtYXRpb246IGh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZS9kaXN0aWwtd2hpc3BlcikuCiAgICAgICAgOnBhcmFtIG1heF9uZXdfdG9rZW5zOiAgICAgICAgICAgIFRoZSBtYXhpbXVtIG51bWJlciBvZiBuZXcgdG9rZW5zIHRvIGdlbmVyYXRlLiBUaGlzIGlzIHVzZWQgdG8gbGltaXQgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGdlbmVyYXRpb24gbGVuZ3RoLiBEZWZhdWx0IGlzIDEyOCB0b2tlbnMuCiAgICAgICAgOnBhcmFtIGNodW5rX2xlbmd0aF9zOiAgICAgICAgICAgIFRoZSBhdWRpbyBjaHVuayB0byBzcGxpdCB0aGUgYXVkaW8gdG8gKGluIHNlY29uZHMpLiBEZWZhdWx0IGlzIDMwIHNlY29uZHMuCiAgICAgICAgOnBhcmFtIGJhdGNoX3NpemU6ICAgICAgICAgICAgICAgIFRoZSBiYXRjaCBzaXplIHRvIHVzZSBmb3IgaW5mZXJlbmNlLiBEZWZhdWx0IGlzIDIuCiAgICAgICAgOnBhcmFtIHNwb2tlbl9sYW5ndWFnZTogICAgICAgICAgIEFpbSB3aGlzcGVyIHRvIGtub3cgd2hhdCBsYW5ndWFnZSBpcyBzcG9rZW4uIElmIE5vbmUsIGl0IHdpbGwgdHJ5IHRvIGRldGVjdCBpdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb3IgZWFjaCBjaHVuay4KICAgICAgICA6cGFyYW0gdHJhbnNsYXRlX3RvX2VuZ2xpc2g6ICAgICAgV2hldGhlciB0byB0cmFuc2xhdGUgdGhlIHRyYW5zY3JpcHRpb25zIHRvIEVuZ2xpc2guIERlZmF1bHQgaXMgRmFsc2UuCiAgICAgICAgOnBhcmFtIHJldHVybl90aW1lc3RhbXBzOiAgICAgICAgIFdoZXRoZXIgdG8gcmV0dXJuIHRoZSB0aW1lc3RhbXBzIG9mIHRoZSB3b3Jkcy4gSWYgIndvcmQiLCB3aWxsIHJldHVybiB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGltZXN0YW1wcyBvZiBlYWNoIHdvcmQuIElmIFRydWUgd2lsbCByZXR1cm4gdGhlIHRpbWVzdGFtcHMgb2YgZWFjaCBjaHVuay4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgRGVmYXVsdCBpcyBGYWxzZS4gQWltZWQgdG8gYmUgdXNlZCBmb3Igc3BlZWNoIGRpYXJpemF0aW9uLgogICAgICAgIDpwYXJhbSBwZXJfY2hhbm5lbF90cmFuc2NyaXB0aW9uOiBXaGV0aGVyIHRvIGRvIHBlciBjaGFubmVsIHRyYW5zY3JpcHRpb24uIElmIG5lZWRlZCB0byBydW4gcGVyIGNoYW5uZWwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhbnNjcmlwdGlvbiwgcGFzcyB0aGUgbnVtYmVyIG9mIGNoYW5uZWxzIGV4cGVjdGVkIGZvciBlYWNoIGF1ZGlvIGZpbGUgaGVyZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgMCBtZWFucyByZWd1bGFyIHRyYW5zY3JpcHRpb24gKG1lcmdlIGNoYW5uZWxzKS4KCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE5vdGU6IElmIGBwZXJfY2hhbm5lbF90cmFuc2NyaXB0aW9uYCBpcyBub3QgMCwgYGJhdGNoX3NpemVgIG11c3QgYmUgdHJlYXRlZCB0bwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB0aGUgbnVtYmVyIG9mIGNoYW5uZWxzIGFuZCBub3QgYXVkaW8gZmlsZXMuIEFpbWVkIHRvIGJlIHVzZWQgZm9yIHBlcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjaGFubmVsIHNwZWVjaCBkaWFyaXphdGlvbi4KICAgICAgICAiIiIKICAgICAgICAjIFN0b3JlIGxvYWRpbmcgcGFyYW1ldGVyczoKICAgICAgICBzZWxmLl9tb2RlbF9uYW1lID0gbW9kZWxfbmFtZQogICAgICAgIHNlbGYuX2RldmljZSA9IGRldmljZQogICAgICAgIHNlbGYuX3VzZV9mbGFzaF9hdHRlbnRpb25fMiA9IHVzZV9mbGFzaF9hdHRlbnRpb25fMgogICAgICAgIHNlbGYuX3VzZV9iZXR0ZXJfdHJhbnNmb3JtZXJzID0gdXNlX2JldHRlcl90cmFuc2Zvcm1lcnMKICAgICAgICBzZWxmLl9tYXhfbmV3X3Rva2VucyA9IG1heF9uZXdfdG9rZW5zCiAgICAgICAgc2VsZi5fY2h1bmtfbGVuZ3RoX3MgPSBjaHVua19sZW5ndGhfcwogICAgICAgIHNlbGYuX2JhdGNoX3NpemUgPSBiYXRjaF9zaXplCiAgICAgICAgc2VsZi5fcmV0dXJuX3RpbWVzdGFtcHMgPSByZXR1cm5fdGltZXN0YW1wcwogICAgICAgIHNlbGYuX3Blcl9jaGFubmVsX3RyYW5zY3JpcHRpb24gPSBwZXJfY2hhbm5lbF90cmFuc2NyaXB0aW9uCgogICAgICAgICMgU3RvcmUgZ2VuZXJhdGlvbiBwYXJhbWV0ZXJzOgogICAgICAgIHNlbGYuX2Fzc2lzdGFudF9tb2RlbCA9IGFzc2lzdGFudF9tb2RlbAogICAgICAgIHNlbGYuX3Nwb2tlbl9sYW5ndWFnZSA9IHNwb2tlbl9sYW5ndWFnZQogICAgICAgIHNlbGYuX3RyYW5zbGF0ZV90b19lbmdsaXNoID0gdHJhbnNsYXRlX3RvX2VuZ2xpc2gKCiAgICAgICAgIyBQcmVwYXJlIHRoZSB0cmFuc2NyaXB0aW9uIG9iamVjdHM6CiAgICAgICAgc2VsZi5fdHJhbnNjcmlwdGlvbl9waXBlbGluZTogQXV0b21hdGljU3BlZWNoUmVjb2duaXRpb25QaXBlbGluZSA9IE5vbmUKICAgICAgICBzZWxmLl9nZW5lcmF0ZV9rd2FyZ3M6IGRpY3QgPSBOb25lCgogICAgZGVmIGxvYWQoc2VsZik6CiAgICAgICAgIiIiCiAgICAgICAgTG9hZCB0aGUgdHJhbnNjcmliZXIuIE11c3QgYmUgY2FsbGVkIGJlZm9yZSB0cmFuc2NyaWJpbmcuCiAgICAgICAgIiIiCiAgICAgICAgIyBTZXQgdGhlIGRldmljZSBhbmQgZGF0YSB0eXBlIHRvIHVzZSAocHJlZmVyIEdQVSBpZiBhdmFpbGFibGUpOgogICAgICAgIGRldmljZSA9IHRvcmNoLmRldmljZSgKICAgICAgICAgICAgc2VsZi5fZGV2aWNlIG9yICJjdWRhIiBpZiB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpIGVsc2UgImNwdSIKICAgICAgICApCiAgICAgICAgdG9yY2hfZHR5cGUgPSB0b3JjaC5mbG9hdDE2IGlmIGRldmljZS50eXBlID09ICJjdWRhIiBlbHNlIHRvcmNoLmZsb2F0MzIKCiAgICAgICAgIyBDaG9vc2UgdGhlIG9wdGltaXphdGlvbiB0byB1c2UgKGluIGNhc2UgdGhlIHVzZXIgZGlkIG5vdCBzcGVjaWZ5IGFueSk6CiAgICAgICAgaWYgKAogICAgICAgICAgICBzZWxmLl91c2VfZmxhc2hfYXR0ZW50aW9uXzIgaXMgTm9uZQogICAgICAgICAgICBhbmQgc2VsZi5fdXNlX2JldHRlcl90cmFuc2Zvcm1lcnMgaXMgTm9uZQogICAgICAgICk6CiAgICAgICAgICAgICMgUHJlZmVyIHRvIHVzZSBmbGFzaCBhdHRlbnRpb24gMiBpZiBhdmFpbGFibGUgYW5kIGN1ZGEgZGV2aWNlIGlzIHN1cHBvcnRlZCAoc2VlIEdQVSBuYW1lcyB0byBhcmNoaXRlY3R1cmUKICAgICAgICAgICAgIyBoZXJlOiBodHRwczovL2VuLndpa2lwZWRpYS5vcmcvd2lraS9MaXN0X29mX052aWRpYV9ncmFwaGljc19wcm9jZXNzaW5nX3VuaXRzI1Rlc2xhKToKICAgICAgICAgICAgaWYgZGV2aWNlLnR5cGUgPT0gImN1ZGEiIGFuZCBpc19mbGFzaF9hdHRuXzJfYXZhaWxhYmxlKCk6CiAgICAgICAgICAgICAgICBjdWRhX2RldmljZV9uYW1lID0gdG9yY2guY3VkYS5nZXRfZGV2aWNlX3Byb3BlcnRpZXMoZGV2aWNlKS5uYW1lCiAgICAgICAgICAgICAgICBpZiBhbnkoCiAgICAgICAgICAgICAgICAgICAgY3VkYV9kZXZpY2VfbmFtZS5zdGFydHN3aXRoKGdwdV9uYW1lKQogICAgICAgICAgICAgICAgICAgIGZvciBncHVfbmFtZSBpbiBbCiAgICAgICAgICAgICAgICAgICAgICAgICJOVklESUEgQSIsICAjIEZvciBBbXBlcmUgYXJjaGl0ZWN0dXJlIChlLmcuIEExMCwgQTMwLCBBMTAwKQogICAgICAgICAgICAgICAgICAgICAgICAiTlZJRElBIEgiLCAgIyBGb3IgSG9wcGVyIGFyY2hpdGVjdHVyZSAoZS5nLiBIMTAwKQogICAgICAgICAgICAgICAgICAgICAgICAiTlZJRElBIEwiLCAgIyBGb3IgQWRhIExvdmVsYWNlIGFyY2hpdGVjdHVyZSAoZS5nLiBMNCwgTDQwKQogICAgICAgICAgICAgICAgICAgICAgICAiTlZJRElBIFJUWCAzMCIsICAjIEZvciBBZGEgTG92ZWxhY2UgYXJjaGl0ZWN0dXJlIChSVFggMzAgc2VyaWVzKQogICAgICAgICAgICAgICAgICAgICAgICAiTlZJRElBIFJUWCA0MCIsICAjIEZvciBBZGEgTG92ZWxhY2UgYXJjaGl0ZWN0dXJlIChSVFggNDAgc2VyaWVzKQogICAgICAgICAgICAgICAgICAgICAgICAiTlZJRElBIFJUWCA1MCIsICAjIEZvciBBZGEgTG92ZWxhY2UgYXJjaGl0ZWN0dXJlIChSVFggNTAgc2VyaWVzKQogICAgICAgICAgICAgICAgICAgICAgICAjIFdpbGwgYmUgc3VwcG9ydGVkIHNvb24gYWNjb3JkaW5nIHRvIEZsYXNoQXR0ZW50aW9uIEdpdEh1YiByZXBvOgogICAgICAgICAgICAgICAgICAgICAgICAjIGh0dHBzOi8vZ2l0aHViLmNvbS9EYW8tQUlMYWIvZmxhc2gtYXR0ZW50aW9uP3RhYj1yZWFkbWUtb3YtZmlsZSNpbnN0YWxsYXRpb24tYW5kLWZlYXR1cmVzCiAgICAgICAgICAgICAgICAgICAgICAgICMgIk5WSURJQSBUNCIsICAjIEZvciBUdXJpbmcgYXJjaGl0ZWN0dXJlIChvbmx5IFQ0KQogICAgICAgICAgICAgICAgICAgICAgICAjICJOVklESUEgUlRYIDIwIiwgICMgRm9yIFR1cmluZyBhcmNoaXRlY3R1cmUgKFJUWCAyMCBzZXJpZXMpCiAgICAgICAgICAgICAgICAgICAgXQogICAgICAgICAgICAgICAgKToKICAgICAgICAgICAgICAgICAgICBzZWxmLl91c2VfZmxhc2hfYXR0ZW50aW9uXzIgPSBUcnVlCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIHNlbGYuX3VzZV9iZXR0ZXJfdHJhbnNmb3JtZXJzID0gVHJ1ZQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgc2VsZi5fdXNlX2JldHRlcl90cmFuc2Zvcm1lcnMgPSBUcnVlCgogICAgICAgICMgQnVpbGQgdGhlIG9wdGltaXphdGlvbnMga3dhcmdzOgogICAgICAgIG1vZGVsX2t3YXJncyA9IHsKICAgICAgICAgICAgImxvd19jcHVfbWVtX3VzYWdlIjogVHJ1ZSwKICAgICAgICAgICAgInVzZV9zYWZldGVuc29ycyI6IFRydWUsCiAgICAgICAgfQogICAgICAgIGlmIHNlbGYuX3VzZV9mbGFzaF9hdHRlbnRpb25fMjoKICAgICAgICAgICAgaWYgX0xPR0dFUjoKICAgICAgICAgICAgICAgIF9MT0dHRVIuaW5mbygKICAgICAgICAgICAgICAgICAgICAiVXNpbmcgRmxhc2hBdHRlbnRpb24yIG9wdGltaXphdGlvbiAtIG1ha2Ugc3VyZSB0aGUgYGZsYXNoLWF0dG5gIHBhY2thZ2UgaXMgaW5zdGFsbGVkIHZpYSAiCiAgICAgICAgICAgICAgICAgICAgImBwaXAgaW5zdGFsbCAtVSBmbGFzaC1hdHRuIC0tbm8tYnVpbGQtaXNvbGF0aW9uYCIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgbW9kZWxfa3dhcmdzWyJhdHRuX2ltcGxlbWVudGF0aW9uIl0gPSAiZmxhc2hfYXR0ZW50aW9uXzIiCiAgICAgICAgZWxpZiBzZWxmLl91c2VfYmV0dGVyX3RyYW5zZm9ybWVyczoKICAgICAgICAgICAgaWYgX0xPR0dFUjoKICAgICAgICAgICAgICAgIF9MT0dHRVIuaW5mbygKICAgICAgICAgICAgICAgICAgICAiVXNpbmcgQmV0dGVyVHJhbnNmb3JtZXJzIG9wdGltaXphdGlvbiAtIG1ha2Ugc3VyZSB0aGUgYG9wdGltdW1gIHBhY2thZ2UgaXMgaW5zdGFsbGVkIHZpYSAiCiAgICAgICAgICAgICAgICAgICAgImBwaXAgaW5zdGFsbCAtVSBvcHRpbXVtYCIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgbW9kZWxfa3dhcmdzWyJhdHRuX2ltcGxlbWVudGF0aW9uIl0gPSAic2RwYSIKCiAgICAgICAgIyBJbml0aWFsaXplIHRoZSBzcGVlY2ggcmVjb2duaXRpb24gcGlwZWxpbmU6CiAgICAgICAgc2VsZi5fdHJhbnNjcmlwdGlvbl9waXBlbGluZSA9IHBpcGVsaW5lKAogICAgICAgICAgICB0YXNrPSJhdXRvbWF0aWMtc3BlZWNoLXJlY29nbml0aW9uIiwKICAgICAgICAgICAgbW9kZWw9c2VsZi5fbW9kZWxfbmFtZSwKICAgICAgICAgICAgbW9kZWxfa3dhcmdzPW1vZGVsX2t3YXJncy5jb3B5KCksCiAgICAgICAgICAgIGJhdGNoX3NpemU9c2VsZi5fYmF0Y2hfc2l6ZSwKICAgICAgICAgICAgbWF4X25ld190b2tlbnM9c2VsZi5fbWF4X25ld190b2tlbnMsCiAgICAgICAgICAgIGNodW5rX2xlbmd0aF9zPXNlbGYuX2NodW5rX2xlbmd0aF9zLAogICAgICAgICAgICByZXR1cm5fdGltZXN0YW1wcz1zZWxmLl9yZXR1cm5fdGltZXN0YW1wcywKICAgICAgICAgICAgdG9yY2hfZHR5cGU9dG9yY2hfZHR5cGUsCiAgICAgICAgICAgIGRldmljZT1kZXZpY2UsCiAgICAgICAgKQoKICAgICAgICAjIFByZXBhcmUgdGhlIGdlbmVyYXRpb24ga3dhcmdzOgogICAgICAgIHNlbGYuX2dlbmVyYXRlX2t3YXJncyA9IHsKICAgICAgICAgICAgImxhbmd1YWdlIjogc2VsZi5fc3Bva2VuX2xhbmd1YWdlLAogICAgICAgICAgICAidGFzayI6ICJ0cmFuc2xhdGUiIGlmIHNlbGYuX3RyYW5zbGF0ZV90b19lbmdsaXNoIGVsc2UgInRyYW5zY3JpYmUiLAogICAgICAgIH0KCiAgICAgICAgIyBJbml0aWFsaXplIHRoZSBhc3Npc3RhbnQgbW9kZWwgKGlmIG5lZWRlZCk6CiAgICAgICAgaWYgc2VsZi5fYXNzaXN0YW50X21vZGVsOgogICAgICAgICAgICBhc3Npc3RhbnRfbW9kZWwgPSBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoCiAgICAgICAgICAgICAgICBzZWxmLl9hc3Npc3RhbnRfbW9kZWwsIHRvcmNoX2R0eXBlPXRvcmNoX2R0eXBlLCAqKm1vZGVsX2t3YXJncwogICAgICAgICAgICApCiAgICAgICAgICAgIGFzc2lzdGFudF9tb2RlbC50byhkZXZpY2UpCiAgICAgICAgICAgIHNlbGYuX2dlbmVyYXRlX2t3YXJnc1siYXNzaXN0YW50X21vZGVsIl0gPSBhc3Npc3RhbnRfbW9kZWwKCiAgICBkZWYgdHJhbnNjcmliZSgKICAgICAgICBzZWxmLAogICAgICAgIGF1ZGlvX2ZpbGVzOiBMaXN0W1BhdGhdLAogICAgICAgIGJhdGNoX3Byb2Nlc3NvcjogQmF0Y2hQcm9jZXNzb3IgPSBOb25lLAogICAgICAgIGJhdGNoZXNfcXVldWU6IFF1ZXVlID0gTm9uZSwKICAgICAgICB2ZXJib3NlOiBib29sID0gRmFsc2UsCiAgICApIC0+IFVuaW9uW0xpc3RbTGlzdFtkaWN0XV0sIE5vbmVdOgogICAgICAgICIiIgogICAgICAgIFRyYW5zY3JpYmUgdGhlIGdpdmVuIGF1ZGlvIGZpbGVzLiBUaGUgdHJhbnNjcmlwdGlvbnMgd2lsbCBiZSBzZW50IHRvIGEgcXVldWUgb3IgYSBiYXRjaCBwcm9jZXNzb3IgZm9yIGZ1cnRoZXIKICAgICAgICBwcm9jZXNzaW5nIGxpa2Ugd3JpdGluZyB0byB0ZXh0IGZpbGVzLiBJZiBubyBxdWV1ZSBvciBiYXRjaCBwcm9jZXNzb3IgaXMgZ2l2ZW4sIHRoZSB0cmFuc2NyaXB0aW9ucyBvdXRwdXRzIGZyb20KICAgICAgICB0aGUgcGlwZWxpbmUgd2lsbCBiZSByZXR1cm5lZC4gT3RoZXJ3aXNlLCBgTm9uZWAgaXMgcmV0dXJuZWQuCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlczogICAgIFRoZSBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlLgogICAgICAgIDpwYXJhbSBiYXRjaF9wcm9jZXNzb3I6IEEgYmF0Y2ggcHJvY2Vzc29yLgogICAgICAgIDpwYXJhbSBiYXRjaGVzX3F1ZXVlOiAgIEEgbXVsdGlwcm9jZXNzaW5nIHF1ZXVlIHRvIHB1dCB0aGUgYmF0Y2hlcyBpbi4KICAgICAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICBXaGV0aGVyIHRvIHNob3cgYSBwcm9ncmVzcyBiYXIuIERlZmF1bHQgaXMgRmFsc2UuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgdHJhbnNjcmlwdGlvbnMgb3V0cHV0cyBmcm9tIHRoZSBwaXBlbGluZSBpZiBubyBxdWV1ZSBvciBiYXRjaCBwcm9jZXNzb3IgaXMgZ2l2ZW4sIG90aGVyd2lzZSwKICAgICAgICAgICAgICAgICAgYE5vbmVgLgogICAgICAgICIiIgogICAgICAgICMgV3JhcCB0aGUgYXVkaW8gZmlsZXMgd2l0aCBhIGZ1bmN0aW9uIHRvIGl0ZXJhdGUgb3ZlciB0aGVtIHZpYSBhIGdlbmVyYXRvciAoc2F2ZSBtZW1vcnkgYW5kIHJ1bnRpbWUgd2l0aAogICAgICAgICMgSHVnZ2luZ2ZhY2UncyBwaXBlbGluZXMgYXMgdGhleSBwcmVsb2FkIGVhY2ggaW5wdXQgd2hpbGUgaW5mZXJlbmNlIGlzIHJ1bm5pbmcpOgogICAgICAgIGRlZiBhdWRpb19pdGVyYXRvcigpIC0+IEdlbmVyYXRvcltVbmlvbltkaWN0LCBzdHJdLCBOb25lLCBOb25lXToKICAgICAgICAgICAgaWYgc2VsZi5fcGVyX2NoYW5uZWxfdHJhbnNjcmlwdGlvbjoKICAgICAgICAgICAgICAgIGZvciBhdWRpb19maWxlIGluIGF1ZGlvX2ZpbGVzOgogICAgICAgICAgICAgICAgICAgIGF1ZGlvLCBzYW1wbGluZ19yYXRlID0gdG9yY2hhdWRpby5sb2FkKHN0cihhdWRpb19maWxlKSkKICAgICAgICAgICAgICAgICAgICBhdWRpbyA9IGF1ZGlvLm51bXB5KCkKICAgICAgICAgICAgICAgICAgICBmb3IgY2hhbm5lbCBpbiBhdWRpbzoKICAgICAgICAgICAgICAgICAgICAgICAgeWllbGQgeyJyYXciOiBjaGFubmVsLCAic2FtcGxpbmdfcmF0ZSI6IHNhbXBsaW5nX3JhdGV9CiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBmb3IgYXVkaW9fZmlsZSBpbiBhdWRpb19maWxlczoKICAgICAgICAgICAgICAgICAgICB5aWVsZCBzdHIoYXVkaW9fZmlsZSkKCiAgICAgICAgIyBDcmVhdGUgYSBiYXRjaCBpdGVyYXRvcjoKICAgICAgICBkZWYgYmF0Y2hfaXRlcmF0b3IoKSAtPiBHZW5lcmF0b3JbTGlzdFtVbmlvbltkaWN0LCBzdHJdXSwgTm9uZSwgTm9uZV06CiAgICAgICAgICAgIGJhdGNoID0gW10KICAgICAgICAgICAgZm9yIGF1ZGlvIGluIGF1ZGlvX2l0ZXJhdG9yKCk6CiAgICAgICAgICAgICAgICBiYXRjaC5hcHBlbmQoYXVkaW8pCiAgICAgICAgICAgICAgICBpZiBsZW4oYmF0Y2gpID09IHNlbGYuX2JhdGNoX3NpemU6CiAgICAgICAgICAgICAgICAgICAgeWllbGQgYmF0Y2gKICAgICAgICAgICAgICAgICAgICBiYXRjaCA9IFtdCiAgICAgICAgICAgIGlmIGJhdGNoOgogICAgICAgICAgICAgICAgeWllbGQgYmF0Y2gKCiAgICAgICAgIyBQcmVwYXJlIHRoZSBzdWNjZXNzZXMgZGF0YWZyYW1lIGFuZCBlcnJvcnMgZGljdGlvbmFyeSB0byBiZSByZXR1cm5lZDoKICAgICAgICBvdXRwdXRzID0gW10KCiAgICAgICAgIyBJbmZlciB0aHJvdWdoIHRoZSBwaXBlbGluZToKICAgICAgICBmb3IgaW5wdXRfYmF0Y2ggaW4gdHFkbSgKICAgICAgICAgICAgYmF0Y2hfaXRlcmF0b3IoKSBpZiBzZWxmLl9iYXRjaF9zaXplID4gMSBlbHNlIGF1ZGlvX2l0ZXJhdG9yKCksCiAgICAgICAgICAgIGRlc2M9IlRyYW5zY3JpYmluZyIsCiAgICAgICAgICAgIHVuaXQ9ImNoYW5uZWwiIGlmIHNlbGYuX3Blcl9jaGFubmVsX3RyYW5zY3JpcHRpb24gZWxzZSAiYXVkaW8gZmlsZSIsCiAgICAgICAgICAgIHRvdGFsPSgKICAgICAgICAgICAgICAgICgKICAgICAgICAgICAgICAgICAgICAobGVuKGF1ZGlvX2ZpbGVzKSAvLyBzZWxmLl9iYXRjaF9zaXplKQogICAgICAgICAgICAgICAgICAgICsgKGxlbihhdWRpb19maWxlcykgJSBzZWxmLl9iYXRjaF9zaXplICE9IDApCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICAqIChzZWxmLl9wZXJfY2hhbm5lbF90cmFuc2NyaXB0aW9uIG9yIDEpCiAgICAgICAgICAgICksCiAgICAgICAgICAgIGRpc2FibGU9bm90IHZlcmJvc2UsCiAgICAgICAgKToKICAgICAgICAgICAgIyBJbmZlcjoKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgb3V0cHV0X2JhdGNoID0gc2VsZi5fdHJhbnNjcmlwdGlvbl9waXBlbGluZSgKICAgICAgICAgICAgICAgICAgICBpbnB1dF9iYXRjaCwKICAgICAgICAgICAgICAgICAgICBnZW5lcmF0ZV9rd2FyZ3M9c2VsZi5fZ2VuZXJhdGVfa3dhcmdzLAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGV4Y2VwdGlvbjoKICAgICAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgZXhjZXB0aW9uOgogICAgICAgICAgICAgICAgb3V0cHV0X2JhdGNoID0gc3RyKGV4Y2VwdGlvbikKICAgICAgICAgICAgICAgICMgQWxpZ24gdG8gYmF0Y2ggc2l6ZToKICAgICAgICAgICAgICAgIG91dHB1dF9iYXRjaCA9ICgKICAgICAgICAgICAgICAgICAgICBbb3V0cHV0X2JhdGNoXSAqIGxlbihpbnB1dF9iYXRjaCkKICAgICAgICAgICAgICAgICAgICBpZiBpc2luc3RhbmNlKGlucHV0X2JhdGNoLCBsaXN0KQogICAgICAgICAgICAgICAgICAgIGVsc2UgW291dHB1dF9iYXRjaF0KICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgIyBUbyBhbGlnbiB3aXRoIGJhdGNoaW5nLCBpZiBiYXRjaCBzaXplIGlzIDEsIHdyYXAgdGhlIG91dHB1dCB3aXRoIGEgbGlzdDoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShvdXRwdXRfYmF0Y2gsIGRpY3QpOgogICAgICAgICAgICAgICAgb3V0cHV0X2JhdGNoID0gW291dHB1dF9iYXRjaF0KICAgICAgICAgICAgIyBJZiBhIGJhdGNoIHByb2Nlc3NvciBpcyBnaXZlbiwgcHJvY2VzcyB0aGUgYmF0Y2g6CiAgICAgICAgICAgIGlmIGJhdGNoX3Byb2Nlc3NvcjoKICAgICAgICAgICAgICAgICMgUHJvY2VzcyBpdCBkaXJlY3RseToKICAgICAgICAgICAgICAgIGJhdGNoX3Byb2Nlc3Nvci5wcm9jZXNzX2JhdGNoKGJhdGNoPW91dHB1dF9iYXRjaCkKICAgICAgICAgICAgICAgIGJhdGNoX3Byb2Nlc3Nvci5kb190YXNrcygpCiAgICAgICAgICAgIGVsaWYgYmF0Y2hlc19xdWV1ZToKICAgICAgICAgICAgICAgICMgT3RoZXJ3aXNlLCBxdWV1ZSB0aGUgYmF0Y2g6CiAgICAgICAgICAgICAgICBiYXRjaGVzX3F1ZXVlLnB1dChvdXRwdXRfYmF0Y2gpCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAjIE90aGVyd2lzZSwgY29sbGVjdCB0aGUgb3V0cHV0IGFzIGlzIHdpdGhvdXQgcHJvY2Vzc2luZzoKICAgICAgICAgICAgICAgIG91dHB1dHMuYXBwZW5kKG91dHB1dF9iYXRjaCkKCiAgICAgICAgIyBDaGVjayBpZiBnaXZlbiBhIG11bHRpcHJvY2Vzc2luZyBxdWV1ZSBvciBhIGJhdGNoIHByb2Nlc3NvcjoKICAgICAgICBpZiBiYXRjaGVzX3F1ZXVlOgogICAgICAgICAgICBiYXRjaGVzX3F1ZXVlLnB1dChfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSykKCiAgICAgICAgcmV0dXJuIG91dHB1dHMgaWYgbm90IGJhdGNoX3Byb2Nlc3NvciBlbHNlIE5vbmUKCgojOiBUaGUgdmFsdWUgdG8gc2VuZCBpbnRvIG11bHRpcHJvY2Vzc2luZyBxdWV1ZXMgdG8gc3RvcCB0aGUgcHJvY2VzczoKX01VTFRJUFJPQ0VTU0lOR19TVE9QX01BUksgPSAiU1RPUCIKCgpkZWYgX211bHRpcHJvY2Vzc2luZ19wcm9jZXNzX2JhdGNoZXMoCiAgICBiYXRjaF9wcm9jZXNzb3I6IEJhdGNoUHJvY2Vzc29yLAogICAgYmF0Y2hlc19xdWV1ZTogUXVldWUsCiAgICB0YXNrc19xdWV1ZTogUXVldWUsCiAgICBuX3Rhc2tfY29tcGxldGVyczogaW50LAopOgogICAgIiIiCiAgICBQcm9jZXNzIHRoZSBiYXRjaGVzIGluIHRoZSBnaXZlbiBiYXRjaGVzIHF1ZXVlIGFuZCBwdXQgdGhlIHRhc2tzIGluIHRoZSBnaXZlbiB0YXNrcyBxdWV1ZS4gVGhlIGZ1bmN0aW9uIHdpbGwgc3RvcAogICAgd2hlbiB0aGUgZ2l2ZW4gYmF0Y2hlcyBxdWV1ZSB3aWxsIHJlY2VpdmUgdGhlIHN0b3AgbWFyay4gSXQgaXMgYWltZWQgdG8gYmUgdXNlZCB3aXRoIG11bHRpcHJvY2Vzc2luZyBhcyBhIHByb2Nlc3MuCgogICAgOnBhcmFtIGJhdGNoX3Byb2Nlc3NvcjogICBBIGJhdGNoIHByb2Nlc3NvciB0byBwcm9jZXNzIHRoZSBiYXRjaGVzLgogICAgOnBhcmFtIGJhdGNoZXNfcXVldWU6ICAgICBBIHF1ZXVlIHRvIGdldCB0aGUgYmF0Y2hlcyBmcm9tLgogICAgOnBhcmFtIHRhc2tzX3F1ZXVlOiAgICAgICBBIHF1ZXVlIHRvIHB1dCB0aGUgdGFza3MgaW4uCiAgICA6cGFyYW0gbl90YXNrX2NvbXBsZXRlcnM6IFRoZSBudW1iZXIgb2YgdGFzayBjb21wbGV0ZXJzIChwcm9jZXNzZXMgdGhhdCBydW4gdGhlIGBfbXVsdGlwcm9jZXNzaW5nX2NvbXBsZXRlX3Rhc2tzYAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmdW5jdGlvbikuIEEgc3RvcCBtYXJrIHdpbGwgYmUgc2VudCB0byB0aGUgdGFza3MgcXVldWUgZm9yIGVhY2ggdGFzayBjb21wbGV0ZXIuCiAgICAiIiIKICAgIHdoaWxlIFRydWU6CiAgICAgICAgIyBHZXQgdGhlIGJhdGNoOgogICAgICAgIGJhdGNoOiBMaXN0W2RpY3RdID0gYmF0Y2hlc19xdWV1ZS5nZXQoKQogICAgICAgIGlmIGJhdGNoID09IF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLOgogICAgICAgICAgICBicmVhawoKICAgICAgICAjIFByb2Nlc3MgdGhlIGJhdGNoOgogICAgICAgIGJhdGNoX3Byb2Nlc3Nvci5wcm9jZXNzX2JhdGNoKGJhdGNoPWJhdGNoKQoKICAgICAgICAjIEdldCB0aGUgdGFza3M6CiAgICAgICAgdGFza3MgPSBiYXRjaF9wcm9jZXNzb3IuZ2V0X3Rhc2tzKCkKCiAgICAgICAgIyBRdWV1ZSB0aGUgdGFza3M6CiAgICAgICAgZm9yIHRhc2sgaW4gdGFza3M6CiAgICAgICAgICAgIHRhc2tzX3F1ZXVlLnB1dCh0YXNrLnRvX3R1cGxlKCkpCgogICAgIyBNYXJrIHRoZSBlbmQgb2YgdGhlIGJhdGNoZXM6CiAgICBmb3IgXyBpbiByYW5nZShuX3Rhc2tfY29tcGxldGVycyk6CiAgICAgICAgdGFza3NfcXVldWUucHV0KF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLKQoKCmRlZiBfbXVsdGlwcm9jZXNzaW5nX2NvbXBsZXRlX3Rhc2tzKHRhc2tzX3F1ZXVlOiBRdWV1ZSwgcmVzdWx0c19xdWV1ZTogUXVldWUpOgogICAgIiIiCiAgICBDb21wbGV0ZSB0aGUgdGFza3MgaW4gdGhlIGdpdmVuIHF1ZXVlIGFuZCBwdXQgdGhlIHJlc3VsdHMgaW4gdGhlIGdpdmVuIHJlc3VsdHMgcXVldWUuIFRoZSBmdW5jdGlvbiB3aWxsIHN0b3Agd2hlbgogICAgdGhlIGdpdmVuIHRhc2tzIHF1ZXVlIHdpbGwgcmVjZWl2ZSB0aGUgc3RvcCBtYXJrLiBJdCBpcyBhaW1lZCB0byBiZSB1c2VkIHdpdGggbXVsdGlwcm9jZXNzaW5nIGFzIGEgcHJvY2Vzcy4KCiAgICA6cGFyYW0gdGFza3NfcXVldWU6ICAgQSBxdWV1ZSB0byBnZXQgdGhlIHRhc2tzIGZyb20uCiAgICA6cGFyYW0gcmVzdWx0c19xdWV1ZTogQSBxdWV1ZSB0byBwdXQgdGhlIHJlc3VsdHMgaW4uCiAgICAiIiIKICAgIHRhc2tzX21hcCA9IHsKICAgICAgICBCYXNlVGFzay5fX25hbWVfXzogQmFzZVRhc2ssCiAgICAgICAgU3BlZWNoRGlhcml6YXRpb25UYXNrLl9fbmFtZV9fOiBTcGVlY2hEaWFyaXphdGlvblRhc2ssCiAgICAgICAgU3BlZWNoRGlhcml6YXRpb25QZXJDaGFubmVsVGFzay5fX25hbWVfXzogU3BlZWNoRGlhcml6YXRpb25QZXJDaGFubmVsVGFzaywKICAgIH0KCiAgICB3aGlsZSBUcnVlOgogICAgICAgICMgR2V0IHRoZSB0YXNrOgogICAgICAgIHRhc2sgPSB0YXNrc19xdWV1ZS5nZXQoKQogICAgICAgIGlmIHRhc2sgPT0gX01VTFRJUFJPQ0VTU0lOR19TVE9QX01BUks6CiAgICAgICAgICAgIGJyZWFrCgogICAgICAgICMgUmVjb25zdHJ1Y3QgdGhlIHRhc2s6CiAgICAgICAgdGFza19jbGFzcywgdGFza19rd2FyZ3MgPSB0YXNrCiAgICAgICAgdGFzayA9IHRhc2tzX21hcFt0YXNrX2NsYXNzXSgqKnRhc2tfa3dhcmdzKQoKICAgICAgICAjIENvbXBsZXRlIHRoZSB0YXNrOgogICAgICAgIHRhc2suZG9fdGFzaygpCiAgICAgICAgcmVzdWx0c19xdWV1ZS5wdXQoKHRhc2suaXNfZmFpbGVkKCksIHRhc2suZ2V0X3Jlc3VsdCgpKSkKCiAgICAjIE1hcmsgdGhlIGVuZCBvZiB0aGUgdGFza3M6CiAgICByZXN1bHRzX3F1ZXVlLnB1dChfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSykKCgojIEdldCB0aGUgZ2xvYmFsIGxvZ2dlcjoKX0xPR0dFUiA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCkKCgpkZWYgb3Blbl9tcGlfaGFuZGxlcigKICAgIHdvcmtlcl9pbnB1dHM6IExpc3Rbc3RyXSwgcm9vdF93b3JrZXJfaW5wdXRzOiBEaWN0W3N0ciwgQW55XSA9IE5vbmUKKToKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBDaGVjayBmb3IgTUxSdW4gYW5kIE9wZW5NUEkgYXZhaWxhYmlsaXR5OgogICAgY29udGV4dCwgY29tbSA9IF9jaGVja19tbHJ1bl9hbmRfb3Blbl9tcGkoKQoKICAgICMgQ2hlY2sgaWYgTUxSdW4gaXMgYXZhaWxhYmxlLCBzZXQgdGhlIGdsb2JhbCBsb2dnZXIgdG8gTUxSdW4nczoKICAgIGlmIGNvbnRleHQ6CiAgICAgICAgX0xPR0dFUiA9IGNvbnRleHQubG9nZ2VyCgogICAgZGVmIGRlY29yYXRvcihoYW5kbGVyKToKICAgICAgICBpZiBjb21tIGlzIE5vbmUgb3IgY29tbS5HZXRfc2l6ZSgpID09IDE6CiAgICAgICAgICAgIHJldHVybiBoYW5kbGVyCgogICAgICAgIEB3cmFwcyhoYW5kbGVyKQogICAgICAgIGRlZiB3cmFwcGVyKCoqa3dhcmdzKToKICAgICAgICAgICAgIyBHZXQgdGhlIG9wZW4gbXBpIGVudmlyb25tZW50IHByb3BlcnRpZXM6CiAgICAgICAgICAgIHNpemUgPSBjb21tLkdldF9zaXplKCkKICAgICAgICAgICAgcmFuayA9IGNvbW0uR2V0X3JhbmsoKQoKICAgICAgICAgICAgIyBHaXZlIHRoZSBjb3JyZWN0IGNodW5rIG9mIHRoZSB3b3JrZXJzIGlucHV0czoKICAgICAgICAgICAgZm9yIHdvcmtlcl9pbnB1dCBpbiB3b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBrd2FyZ3Nbd29ya2VyX2lucHV0XQogICAgICAgICAgICAgICAgaWYgaW5wdXRfYXJndW1lbnQgaXMgTm9uZToKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgc3RyKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IF9nZXRfYXVkaW9fZmlsZXMoCiAgICAgICAgICAgICAgICAgICAgICAgIGRhdGFfcGF0aD1QYXRoKGlucHV0X2FyZ3VtZW50KS5hYnNvbHV0ZSgpCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgbGVuKGlucHV0X2FyZ3VtZW50KSA8IHNpemU6CiAgICAgICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAgICAgZiJDYW5ub3Qgc3BsaXQgdGhlIGlucHV0ICd7d29ya2VyX2lucHV0fScgb2YgbGVuZ3RoIHtsZW4oaW5wdXRfYXJndW1lbnQpfSB0byB7c2l6ZX0gd29ya2Vycy4gIgogICAgICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSByZWR1Y2UgdGhlIGFtb3VudCBvZiB3b3JrZXJzIGZvciB0aGlzIGlucHV0LiIKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBldmVuX2NodW5rX3NpemUgPSBsZW4oaW5wdXRfYXJndW1lbnQpIC8vIHNpemUKICAgICAgICAgICAgICAgIGNodW5rX3N0YXJ0ID0gcmFuayAqIGV2ZW5fY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgY2h1bmtfZW5kID0gKAogICAgICAgICAgICAgICAgICAgIChyYW5rICsgMSkgKiBldmVuX2NodW5rX3NpemUKICAgICAgICAgICAgICAgICAgICBpZiByYW5rICsgMSA8IHNpemUKICAgICAgICAgICAgICAgICAgICBlbHNlIGxlbihpbnB1dF9hcmd1bWVudCkKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSYW5rICN7cmFua306IFByb2Nlc3NpbmcgaW5wdXQgY2h1bmsgb2YgJ3t3b3JrZXJfaW5wdXR9JyAiCiAgICAgICAgICAgICAgICAgICAgZiJmcm9tIGluZGV4IHtjaHVua19zdGFydH0gdG8ge2NodW5rX2VuZH0uIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgbGlzdCk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudFtjaHVua19zdGFydDpjaHVua19lbmRdCiAgICAgICAgICAgICAgICBlbGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIHBkLkRhdGFGcmFtZSk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudC5pbG9jW2NodW5rX3N0YXJ0OmNodW5rX2VuZDosIDpdCiAgICAgICAgICAgICAgICBrd2FyZ3Nbd29ya2VyX2lucHV0XSA9IGlucHV0X2FyZ3VtZW50CgogICAgICAgICAgICAjIFNldCB0aGUgcm9vdCB3b3JrZXIgb25seSBhcmd1bWVudHM6CiAgICAgICAgICAgIGlmIHJhbmsgPT0gMCBhbmQgcm9vdF93b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAga3dhcmdzLnVwZGF0ZShyb290X3dvcmtlcl9pbnB1dHMpCgogICAgICAgICAgICAjIFJ1biB0aGUgd29ya2VyOgogICAgICAgICAgICBvdXRwdXQgPSBoYW5kbGVyKCoqa3dhcmdzKQoKICAgICAgICAgICAgIyBTYXZlIHRoZSBvdXRwdXQgZGlyZWN0b3J5IG9mIHRoaXMgd29ya2VyOgogICAgICAgICAgICBvdXRwdXRfZGlyZWN0b3J5ID0gUGF0aChvdXRwdXRbMF0pCgogICAgICAgICAgICAjIFNlbmQgdGhlIG91dHB1dCB0byB0aGUgcm9vdCByYW5rIChyYW5rICMwKToKICAgICAgICAgICAgb3V0cHV0ID0gY29tbS5nYXRoZXIob3V0cHV0LCByb290PTApCgogICAgICAgICAgICAjIEpvaW4gdGhlIGRhdGEgZnJvbSBhbGwgd29ya2VyczoKICAgICAgICAgICAgaWYgcmFuayA9PSAwOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiQ29sbGVjdGluZyBkYXRhIGZyb20gd29ya2VycyB0byByb290IHdvcmtlci4iKQoKICAgICAgICAgICAgICAgICMgQ2hlY2sgaWYgdGhlcmUgYXJlIGRpZmZlcmVudCBvdXRwdXQgZGlyZWN0b3JpZXM6CiAgICAgICAgICAgICAgICBvdXRwdXRfZGlyZWN0b3JpZXMgPSBzZXQoW1BhdGgob3V0X2RpcikgZm9yIG91dF9kaXIsIF8sIF8gaW4gb3V0cHV0XSkKICAgICAgICAgICAgICAgIGZvciByIGluIHJhbmdlKDEsIHNpemUpOgogICAgICAgICAgICAgICAgICAgICMgVHJ1ZSBtZWFucyB0aGUgb3RoZXIgd29ya2VycyBzaG91bGQgcGFzcyB0aGVpciBmaWxlcyB0byB0aGUgcm9vdCB3b3JrZXIgKHJhbmsgMCk6CiAgICAgICAgICAgICAgICAgICAgY29tbS5zZW5kKGxlbihvdXRwdXRfZGlyZWN0b3JpZXMpICE9IDEsIGRlc3Q9cikKCiAgICAgICAgICAgICAgICAjIElmIHRoZXJlIGFyZSBkaWZmZXJlbnQgb3V0cHV0IGRpcmVjdG9yaWVzLCBsaXN0ZW4gdG8gdGhlIG90aGVyIHdvcmtlcnM6CiAgICAgICAgICAgICAgICBpZiBsZW4ob3V0cHV0X2RpcmVjdG9yaWVzKSAhPSAxOgogICAgICAgICAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgZmlsZXMgZnJvbSB0aGUgb3RoZXIgd29ya2VyczoKICAgICAgICAgICAgICAgICAgICBmaWxlcyA9IFtdCiAgICAgICAgICAgICAgICAgICAgZm9yIHIgaW4gcmFuZ2UoMSwgc2l6ZSk6CiAgICAgICAgICAgICAgICAgICAgICAgIGZpbGVzLmV4dGVuZChjb21tLnJlY3Yoc291cmNlPXIpKQogICAgICAgICAgICAgICAgICAgICMgV3JpdGUgdGhlIGZpbGVzIHRvIHRoZSByb290IHdvcmtlcidzIG91dHB1dCBkaXJlY3Rvcnk6CiAgICAgICAgICAgICAgICAgICAgZm9yIGZpbGVfbmFtZSwgZmlsZV9jb250ZW50IGluIGZpbGVzOgogICAgICAgICAgICAgICAgICAgICAgICB3aXRoIG9wZW4ob3V0cHV0X2RpcmVjdG9yeSAvIGZpbGVfbmFtZSwgInciKSBhcyBmOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgZi53cml0ZShmaWxlX2NvbnRlbnQpCgogICAgICAgICAgICAgICAgIyBDb25jYXRlbmF0ZSB0aGUgZGF0YWZyYW1lczoKICAgICAgICAgICAgICAgIGRhdGFmcmFtZSA9IHBkLmNvbmNhdChvYmpzPVtkZiBmb3IgXywgZGYsIF8gaW4gb3V0cHV0XSwgYXhpcz0wKQoKICAgICAgICAgICAgICAgICMgQ29uY2F0ZW5hdGUgdGhlIGVycm9ycyBkaWN0aW9uYXJpZXM6CiAgICAgICAgICAgICAgICBlcnJvcnNfZGljdGlvbmFyeSA9IHJlZHVjZSgKICAgICAgICAgICAgICAgICAgICBvcGVyYXRvci5pb3IsIFtlcnIgZm9yIF8sIF8sIGVyciBpbiBvdXRwdXRdLCB7fQogICAgICAgICAgICAgICAgKQoKICAgICAgICAgICAgICAgIHJldHVybiBzdHIob3V0cHV0X2RpcmVjdG9yeSksIGRhdGFmcmFtZSwgZXJyb3JzX2RpY3Rpb25hcnkKCiAgICAgICAgICAgICMgTGlzdGVuIHRvIHJhbmsgMCB0byBzZWUgaWYgdGhlcmUgYXJlIGRpZmZlcmVudCBvdXRwdXQgZGlyZWN0b3JpZXMgYW5kIHRoaXMgcmFuayBuZWVkIHRvIHNlbmQgaXRzIGZpbGVzIHRvCiAgICAgICAgICAgICMgaXQ6CiAgICAgICAgICAgIGlmIGNvbW0ucmVjdihzb3VyY2U9MCk6CiAgICAgICAgICAgICAgICBmaWxlcyA9IFtdCiAgICAgICAgICAgICAgICBmb3IgZmlsZSBpbiBvcy5saXN0ZGlyKG91dHB1dF9kaXJlY3RvcnkpOgogICAgICAgICAgICAgICAgICAgIHdpdGggb3BlbihvdXRwdXRfZGlyZWN0b3J5IC8gZmlsZSwgInIiKSBhcyBmOgogICAgICAgICAgICAgICAgICAgICAgICBmaWxlcy5hcHBlbmQoKGZpbGUsIGYucmVhZCgpKSkKICAgICAgICAgICAgICAgIGNvbW0uc2VuZChmaWxlcywgZGVzdD0wKQogICAgICAgICAgICByZXR1cm4gTm9uZQoKICAgICAgICByZXR1cm4gd3JhcHBlcgoKICAgIHJldHVybiBkZWNvcmF0b3IKCgpkZWYgX2NoZWNrX21scnVuX2FuZF9vcGVuX21waSgpIC0+IFR1cGxlWyJtbHJ1bi5NTENsaWVudEN0eCIsICJtcGk0cHkuTVBJLkludHJhY29tbSJdOgogICAgaXNfbXBpID0gRmFsc2UKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KCiAgICAgICAgY29udGV4dCA9IG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KG5hbWU9Im1scnVuIikKICAgICAgICBpc19tcGkgPSBjb250ZXh0LmxhYmVscy5nZXQoImtpbmQiLCAiam9iIikgPT0gIm1waWpvYiIKCiAgICAgICAgaWYgaXNfbXBpOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBmcm9tIG1waTRweSBpbXBvcnQgTVBJCgogICAgICAgICAgICAgICAgcmV0dXJuIGNvbnRleHQsIE1QSS5DT01NX1dPUkxECiAgICAgICAgICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yIGFzIG1waTRweV9ub3RfZm91bmQ6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgICAgICAgICAiVG8gZGlzdHJpYnV0ZSB0aGUgZnVuY3Rpb24gdXNpbmcgTUxSdW4ncyAnbXBpam9iJyB5b3UgbmVlZCB0byBoYXZlIGBtcGk0cHlgIHBhY2thZ2UgaW4geW91ciAiCiAgICAgICAgICAgICAgICAgICAgImludGVycHJldGVyLiBQbGVhc2UgcnVuIGBwaXAgaW5zdGFsbCBtcGk0cHlgIGFuZCBtYWtlIHN1cmUgeW91IGhhdmUgb3Blbi1tcGkuIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgcmFpc2UgbXBpNHB5X25vdF9mb3VuZAogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJldHVybiBjb250ZXh0LCBOb25lCiAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtb2R1bGVfbm90X2ZvdW5kOgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgcmFpc2UgbW9kdWxlX25vdF9mb3VuZAogICAgcmV0dXJuIE5vbmUsIE5vbmUKCgpAb3Blbl9tcGlfaGFuZGxlcih3b3JrZXJfaW5wdXRzPVsiZGF0YV9wYXRoIl0sIHJvb3Rfd29ya2VyX2lucHV0cz17InZlcmJvc2UiOiBUcnVlfSkKZGVmIHRyYW5zY3JpYmUoCiAgICAjIElucHV0IC8gT3V0cHV0IGt3YXJnczoKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBQYXRoLCBMaXN0W1VuaW9uW3N0ciwgUGF0aF1dXSwKICAgIG91dHB1dF9kaXJlY3Rvcnk6IHN0ciA9IE5vbmUsCiAgICAjIE1vZGVsIGxvYWRpbmcga3dhcmdzOgogICAgbW9kZWxfbmFtZTogc3RyID0gIm9wZW5haS93aGlzcGVyLXRpbnkiLAogICAgZGV2aWNlOiBzdHIgPSBOb25lLAogICAgdXNlX2ZsYXNoX2F0dGVudGlvbl8yOiBib29sID0gTm9uZSwKICAgIHVzZV9iZXR0ZXJfdHJhbnNmb3JtZXJzOiBib29sID0gTm9uZSwKICAgICMgR2VuZXJhdGlvbiBrd2FyZ3M6CiAgICBhc3Npc3RhbnRfbW9kZWw6IHN0ciA9IE5vbmUsCiAgICBtYXhfbmV3X3Rva2VuczogaW50ID0gMTI4LAogICAgY2h1bmtfbGVuZ3RoX3M6IGludCA9IDMwLAogICAgYmF0Y2hfc2l6ZTogaW50ID0gOCwKICAgIHNwb2tlbl9sYW5ndWFnZTogc3RyID0gTm9uZSwKICAgIHRyYW5zbGF0ZV90b19lbmdsaXNoOiBib29sID0gRmFsc2UsCiAgICAjIERpYXJpemF0aW9uIGt3YXJnczoKICAgIHNwZWVjaF9kaWFyaXphdGlvbjogRGljdFtzdHIsIExpc3RbVHVwbGVbZmxvYXQsIGZsb2F0LCBzdHJdXV0gPSBOb25lLAogICAgc3BlZWNoX2RpYXJpemVfcGVyX2NoYW5uZWw6IGludCA9IE5vbmUsCiAgICBzcGVha2VyX2xhYmVsczogTGlzdFtzdHJdID0gTm9uZSwKICAgICMgT3RoZXIga3dhcmdzOgogICAgdXNlX211bHRpcHJvY2Vzc2luZzogVW5pb25bYm9vbCwgaW50XSA9IEZhbHNlLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBUcmFuc2NyaWJlIGF1ZGlvIGZpbGVzIGludG8gdGV4dCBmaWxlcyBhbmQgY29sbGVjdCBhZGRpdGlvbmFsIGRhdGEuIFRoZSBlbmQgcmVzdWx0IGlzIGEgZGlyZWN0b3J5IG9mIHRyYW5zY3JpYmVkCiAgICB0ZXh0IGZpbGVzIGFuZCBhIGRhdGFmcmFtZSBjb250YWluaW5nIHRoZSBmb2xsb3dpbmcgY29sdW1uczoKCiAgICAqIGF1ZGlvX2ZpbGUgLSBUaGUgYXVkaW8gZmlsZSBwYXRoLgogICAgKiB0cmFuc2NyaXB0aW9uX2ZpbGUgLSBUaGUgdHJhbnNjcmliZWQgdGV4dCBmaWxlIG5hbWUgaW4gdGhlIG91dHB1dCBkaXJlY3RvcnkuCgogICAgVGhlIHRyYW5zY3JpcHRpb24gaXMgYmFzZWQgb24gSHVnZ2luZ2ZhY2UncyBBU1IgcGlwZWxpbmUgLQogICAgaHR0cHM6Ly9odWdnaW5nZmFjZS5jby90cmFuc2Zvcm1lcnMvbWFpbl9jbGFzc2VzL3BpcGVsaW5lcy5odG1sI3RyYW5zZm9ybWVycy5BdXRvbWF0aWNTcGVlY2hSZWNvZ25pdGlvblBpcGVsaW5lIGFuZAogICAgaXMgdGVzdGVkIHdpdGggT3BlbkFJJ3MgV2hpc3BlciBtb2RlbHMgLSBodHRwczovL2h1Z2dpbmdmYWNlLmNvL29wZW5haS4KCiAgICBJZiBvbmUgb2YgdGhlIHNwZWFrZXIgZGlhcml6YXRpb24gcGFyYW1ldGVycyBhcmUgZ2l2ZW4gKGVpdGhlciBgc3BlZWNoX2RpYXJpemF0aW9uYCBvcgogICAgYHNwZWVjaF9kaWFyaXplX3Blcl9jaGFubmVsYCksIHRoZSB0cmFuc2NyaXB0aW9uIHdpbGwgYmUgd3JpdHRlbiBpbiBhIGNvbnZlcnNhdGlvbiBmb3JtYXQsIHdoZXJlIGVhY2ggc3BlYWtlciB3aWxsCiAgICBiZSB3cml0dGVuIGluIGEgc2VwYXJhdGUgbGluZTo6CgogICAgICAgIHNwZWFrZXJfMTogdGV4dAogICAgICAgIHNwZWFrZXJfMjogdGV4dAogICAgICAgIHNwZWFrZXJfMTogdGV4dAogICAgICAgIC4uLgoKICAgIDpwYXJhbSBkYXRhX3BhdGg6ICAgICAgICAgICAgICAgICAgQSBkaXJlY3Rvcnkgb2YgYXVkaW8gZmlsZXMgb3IgYSBzaW5nbGUgZmlsZSBvciBhIGxpc3Qgb2YgZmlsZXMgdG8gdHJhbnNjcmliZS4KICAgIDpwYXJhbSBvdXRwdXRfZGlyZWN0b3J5OiAgICAgICAgICAgUGF0aCB0byBhIGRpcmVjdG9yeSB0byBzYXZlIGFsbCB0cmFuc2NyaWJlZCBhdWRpbyBmaWxlcy4gSWYgbm90IGdpdmVuLCB3aWxsIHNhdmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlIHRyYW5zY3JpYmVkIGZpbGVzIGluIGEgdGVtcG9yYXJ5IGRpcmVjdG9yeS4KICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIG5hbWUgdG8gdXNlLiBTaG91bGQgYmUgYSBtb2RlbCBmcm9tIHRoZSBPcGVuQUkncyBXaGlzcGVyIG1vZGVscyBmb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYmVzdCByZXN1bHRzIChmb3IgZXhhbXBsZSAidGlueSIsICJiYXNlIiwgImxhcmdlIiwgZXRjLikuIFNlZSBoZXJlIGZvciBtb3JlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGluZm9ybWF0aW9uOiBodHRwczovL2h1Z2dpbmdmYWNlLmNvL29wZW5haT9zZWFyY2hfbW9kZWxzPXdoaXNwZXIuCiAgICA6cGFyYW0gZGV2aWNlOiAgICAgICAgICAgICAgICAgICAgIFRoZSBkZXZpY2UgdG8gdXNlIGZvciBpbmZlcmVuY2UuIElmIG5vdCBnaXZlbiwgd2lsbCB1c2UgR1BVIGlmIGF2YWlsYWJsZS4KICAgIDpwYXJhbSB1c2VfZmxhc2hfYXR0ZW50aW9uXzI6ICAgICAgV2hldGhlciB0byB1c2UgdGhlIEZsYXNoIEF0dGVudGlvbiAyIGltcGxlbWVudGF0aW9uLiBJdCBjYW4gYmUgdXNlZCBvbmx5IHdpdGgKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb25lIG9mIHRoZSBmb2xsb3dpbmcgR1BVczogTnZpZGlhIEggc2VyaWVzIGFuZCBOdmlkaWEgQSBzZXJpZXMuIFQ0IHN1cHBvcnQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSBhdmFpbGFibGUgc29vbi4KCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE5vdGU6IElmIGJvdGggYHVzZV9mbGFzaF9hdHRlbnRpb25fMmAgYW5kCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGB1c2VfYmV0dGVyX3RyYW5zZm9ybWVyc2AgYXJlIGBOb25lYCwgdGhlIG9wdGltaXphdGlvbiB3aWxsIGJlIGNob3NlbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhdXRvbWF0aWNhbGx5IGFjY29yZGluZyB0byB0aGUgYXZhaWxhYmxlIHJlc291cmNlcy4KCiAgICA6cGFyYW0gdXNlX2JldHRlcl90cmFuc2Zvcm1lcnM6ICAgIFdoZXRoZXIgdG8gdXNlIHRoZSBCZXR0ZXIgVHJhbnNmb3JtZXJzIGxpYnJhcnkgdG8gZnVydGhlciBvcHRpbWl6ZSB0aGUgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFNob3VsZCBiZSB1c2VkIGZvciBhbGwgdXNlIGNhc2VzIHRoYXQgZG8gbm90IHN1cHBvcnQgZmxhc2ggYXR0ZW50aW9uIDIuCgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBOb3RlOiBJZiBib3RoIGB1c2VfZmxhc2hfYXR0ZW50aW9uXzJgIGFuZCBgdXNlX2JldHRlcl90cmFuc2Zvcm1lcnNgIGFyZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgTm9uZWAsIHRoZSBvcHRpbWl6YXRpb24gd2lsbCBiZSBjaG9zZW4gYXV0b21hdGljYWxseSBhY2NvcmRpbmcgdG8gdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGF2YWlsYWJsZSByZXNvdXJjZXMuCiAgICA6cGFyYW0gYXNzaXN0YW50X21vZGVsOiAgICAgICAgICAgIFRoZSBhc3Npc3RhbnQgbW9kZWwgbmFtZSB0byB1c2UgZm9yIGluZmVyZW5jZS4gTm90aWNlIHRoYXQgdGhlIG9wdGltaXphdGlvbnMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKGZsYXNoIGF0dGVudGlvbiAyIGFuZCBiZXR0ZXIgdHJhbnNmb3JtZXJzKSB3aWxsIGJlIGFwcGxpZWQgZm9yIHRoZSBhc3Npc3RhbnQgYXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2VsbC4gU2hvdWxkIGJlIGEgbW9kZWwgZnJvbSBIdWdnaW5nZmFjZSdzIGRpc3RpbC13aGlzcGVyIChzZWUgaGVyZSBmb3IgbW9yZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpbmZvcm1hdGlvbjogaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2Rpc3RpbC13aGlzcGVyKS4KCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE5vdGU6IEN1cnJlbnRseSBhbiBhc3Npc3RhbnQgbW9kZWwgaXMgb25seSB1c2FibGUgd2l0aCBiYXRjaCBzaXplIG9mIDEuCiAgICA6cGFyYW0gbWF4X25ld190b2tlbnM6ICAgICAgICAgICAgIFRoZSBtYXhpbXVtIG51bWJlciBvZiBuZXcgdG9rZW5zIHRvIGdlbmVyYXRlLiBUaGlzIGlzIHVzZWQgdG8gbGltaXQgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGdlbmVyYXRpb24gbGVuZ3RoLiBEZWZhdWx0IGlzIDEyOCB0b2tlbnMuCiAgICA6cGFyYW0gY2h1bmtfbGVuZ3RoX3M6ICAgICAgICAgICAgIFRoZSBhdWRpbyBjaHVuayB0byBzcGxpdCB0aGUgYXVkaW8gdG8gKGluIHNlY29uZHMpLiBEZWZhdWx0IGlzIDMwIHNlY29uZHMuCiAgICA6cGFyYW0gYmF0Y2hfc2l6ZTogICAgICAgICAgICAgICAgIFRoZSBiYXRjaCBzaXplIHRvIHVzZSBmb3IgaW5mZXJlbmNlLiBEZWZhdWx0IGlzIDIuCiAgICA6cGFyYW0gc3Bva2VuX2xhbmd1YWdlOiAgICAgICAgICAgIEFpbSB3aGlzcGVyIHRvIGtub3cgd2hhdCBsYW5ndWFnZSBpcyBzcG9rZW4uIElmIE5vbmUsIGl0IHdpbGwgdHJ5IHRvIGRldGVjdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpdC4KICAgIDpwYXJhbSB0cmFuc2xhdGVfdG9fZW5nbGlzaDogICAgICAgV2hldGhlciB0byB0cmFuc2xhdGUgdGhlIHRyYW5zY3JpcHRpb25zIHRvIEVuZ2xpc2guCiAgICA6cGFyYW0gc3BlZWNoX2RpYXJpemF0aW9uOiAgICAgICAgIEEgc3BlZWNoIGRpYXJpemF0aW9uIGRpY3Rpb25hcnkgd2l0aCB0aGUgZmlsZSBuYW1lcyB0byB0cmFuc2NyaWJlIGFzIGtleXMgYW5kCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZWlyIGRpYXJpemF0aW9uIGFzIHZhbHVlLiBUaGUgZGlhcml6YXRpb24gaXMgYSBsaXN0IG9mIHR1cGxlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKHN0YXJ0LCBlbmQsIHNwZWFrZXIpLiBBbiBleGFtcGxlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvciBhIGRpYXJpemF0aW9uIGRpY3Rpb25hcnk6OgoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImF1ZGlvX2ZpbGVfbmFtZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzdGFydCI6IDAuMCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVuZCI6IDIuMCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNwZWFrZXIiOiAiQWdlbnQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAic3RhcnQiOiAyLjAsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlbmQiOiA0LjAsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzcGVha2VyIjogIkNsaWVudCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90ZTogVGhlIGRpYXJpemF0aW9uIG11c3QgYmUgZm9yIHRoZSBlbnRpcmUgZHVyYXRpb24gb2YgdGhlIGF1ZGlvIGZpbGUgKGFzIGxvbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYXMgV2hpc3BlciBpcyBwcmVkaWN0aW5nIHdvcmRzIHVwIHVudGlsIHRoZW4uCiAgICA6cGFyYW0gc3BlZWNoX2RpYXJpemVfcGVyX2NoYW5uZWw6IFBlcmZvcm0gc3BlZWNoIGRpYXJpemF0aW9uIHBlciBjaGFubmVsLiBFYWNoIHNwZWFrZXIgaXMgZXhwZWN0ZWQgdG8gYmVsb25nIHRvCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGEgc2VwYXJhdGUgY2hhbm5lbCBpbiB0aGUgYXVkaW8uIE5vdGljZTogVGhpcyB3aWxsIG1ha2UgdGhlIHRyYW5zY3JpcHRpb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2xvd2VyIGFzIGVhY2ggY2hhbm5lbCB3aWwgYmUgdHJhbnNjcmliZWQgc2VwYXJhdGx5LiBJZiBhIHNwZWVjaCBkaWFyaXphdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpcyBwYXNzZWQgKHZpYSB0aGUgYHNwZWVjaF9kaWFyaXphdGlvbmAgcGFyYW1ldGVyKSwgdGhpcyBwYXJhbWV0ZXIgaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaWdub3JlZC4KICAgIDpwYXJhbSBzcGVha2VyX2xhYmVsczogICAgICAgICAgICAgQSBsaXN0IG9mIHNwZWFrZXIgbGFiZWxzIGJ5IGNoYW5uZWwgb3JkZXIgdG8gdXNlIGZvciB3cml0aW5nIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0cmFuc2NyaXB0aW9uIHdpdGggcmVzcGVjdCB0byBwZXIgY2hhbm5lbCBzcGVlY2ggZGlhcml6YXRpb24uIFRoaXMgd29uJ3QgYmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdXNlZCB0b2dldGhlciB3aXRoIGEgZ2l2ZW4gc3BlZWNoIGRpYXJpemF0aW9uICh2aWEgdGhlIGBzcGVlY2hfZGlhcml6YXRpb25gCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhcmFtZXRlcikuCiAgICA6cGFyYW0gdXNlX211bHRpcHJvY2Vzc2luZzogICAgICAgIFdoZXRoZXIgdG8gdXNlIG11bHRpcHJvY2Vzc2luZyB0byB0cmFuc2NyaWJlIHRoZSBhdWRpbyBmaWxlcy4gQ2FuIGJlIGVpdGhlciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJvb2xlYW4gdmFsdWUgb3IgYW4gaW50ZWdlci4gSWYgYFRydWVgLCB3aWxsIHVzZSB0aGUgZGVmYXVsdCBhbW91bnQgb2Ygd29ya2VycwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoMyk6IDEgZm9yIHRyYW5zY3JpcHRpb24sIDEgZm9yIGJhdGNoIHByb2Nlc3NpbmcgYW5kIDEgZm9yIHRhc2sgY29tcGxldGlvbiAoc3VjaAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhcyBzcGVlY2ggZGlhcml6YXRpb24gYW5kIHdyaXRpbmcgdG8gZmlsZXMpLiBUbyBjb250cm9sIHRoZSBhbW91bnQgb2YgdGFza3MKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY29tcGxldGlvbiB3b3JrZXJzLCBhbiBpbnRlZ2VyIGNhbiBiZSBwcm92aWRlZCB0byBzcGVjaWZ5IHRoZSBhbW91bnQgb2Ygd29ya2Vycy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYEZhbHNlYCwgd2lsbCB1c2UgYSBzaW5nbGUgcHJvY2Vzcy4gRGVmYXVsdCBpcyBgRmFsc2VgLgogICAgOnBhcmFtIHZlcmJvc2U6ICAgICAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHByaW50IHRoZSBwcm9ncmVzcyBvZiB0aGUgdHJhbnNjcmlwdGlvbi4gRGVmYXVsdCBpcyBgRmFsc2VgLgogICAgIiIiCiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgR2V0IHRoZSBpbnB1dCBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgYXVkaW8gZmlsZXMuIikKICAgIGF1ZGlvX2ZpbGVzID0gX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbihhdWRpb19maWxlcyl9IGF1ZGlvIGZpbGVzLiIpCgogICAgIyBHZXQgdGhlIG91dHB1dCBkaXJlY3Rvcnk6CiAgICBpZiBvdXRwdXRfZGlyZWN0b3J5IGlzIE5vbmU6CiAgICAgICAgaWYgdmVyYm9zZToKICAgICAgICAgICAgX0xPR0dFUi5pbmZvKCJObyBvdXRwdXQgZGlyZWN0b3J5IGdpdmVuLCB1c2luZyB0ZW1wb3JhcnkgZGlyZWN0b3J5LiIpCiAgICAgICAgb3V0cHV0X2RpcmVjdG9yeSA9IHRlbXBmaWxlLm1rZHRlbXAoKQogICAgb3V0cHV0X2RpcmVjdG9yeSA9IFBhdGgob3V0cHV0X2RpcmVjdG9yeSkuYWJzb2x1dGUoKQogICAgb3V0cHV0X2RpcmVjdG9yeS5ta2RpcihleGlzdF9vaz1UcnVlLCBwYXJlbnRzPVRydWUpCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIlRyYW5zY3JpcHRpb25zIHdpbGwgYmUgc2F2ZWQgdG86IHtvdXRwdXRfZGlyZWN0b3J5fSIpCgogICAgIyBJbml0aWFsaXplIGEgYmF0Y2ggcHJvY2Vzc29yIGFjY29yZGluZyB0byB1c2VyIHJlcXVpcmVtZW50cyAobm8gc3BlZWNoIGRpYXJpemF0aW9uLCBnaXZlbiBzcGVlY2ggZGlhcml6YXRpb24sCiAgICAjIHNwZWVjaCBkaWFyaXphdGlvbiBwZXIgY2hhbm5lbCk6CiAgICBpZiBzcGVlY2hfZGlhcml6YXRpb246CiAgICAgICAgYmF0Y2hfcHJvY2Vzc29yID0gU3BlZWNoRGlhcml6YXRpb25CYXRjaFByb2Nlc3NvcigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIG91dHB1dF9kaXJlY3Rvcnk9b3V0cHV0X2RpcmVjdG9yeSwKICAgICAgICAgICAgc3BlZWNoX2RpYXJpemF0aW9uPXNwZWVjaF9kaWFyaXphdGlvbiwKICAgICAgICApCiAgICBlbGlmIHNwZWVjaF9kaWFyaXplX3Blcl9jaGFubmVsOgogICAgICAgIGJhdGNoX3Byb2Nlc3NvciA9IFBlckNoYW5uZWxTcGVlY2hEaWFyaXphdGlvbkJhdGNoUHJvY2Vzc29yKAogICAgICAgICAgICBhdWRpb19maWxlcz1hdWRpb19maWxlcywKICAgICAgICAgICAgb3V0cHV0X2RpcmVjdG9yeT1vdXRwdXRfZGlyZWN0b3J5LAogICAgICAgICAgICBuX2NoYW5uZWxzPXNwZWVjaF9kaWFyaXplX3Blcl9jaGFubmVsLAogICAgICAgICAgICBzcGVha2Vycz1zcGVha2VyX2xhYmVscywKICAgICAgICApCiAgICBlbHNlOgogICAgICAgIGJhdGNoX3Byb2Nlc3NvciA9IEJhdGNoUHJvY2Vzc29yKAogICAgICAgICAgICBhdWRpb19maWxlcz1hdWRpb19maWxlcywKICAgICAgICAgICAgb3V0cHV0X2RpcmVjdG9yeT1vdXRwdXRfZGlyZWN0b3J5LAogICAgICAgICkKCiAgICAjIEluaXRpYWxpemUgdGhlIHRyYW5zY3JpcHRpb24gcGlwZWxpbmU6CiAgICB0cmFuc2NyaWJlciA9IFRyYW5zY3JpYmVyKAogICAgICAgIGRldmljZT1kZXZpY2UsCiAgICAgICAgdXNlX2ZsYXNoX2F0dGVudGlvbl8yPXVzZV9mbGFzaF9hdHRlbnRpb25fMiwKICAgICAgICB1c2VfYmV0dGVyX3RyYW5zZm9ybWVycz11c2VfYmV0dGVyX3RyYW5zZm9ybWVycywKICAgICAgICBhc3Npc3RhbnRfbW9kZWw9YXNzaXN0YW50X21vZGVsLAogICAgICAgIG1vZGVsX25hbWU9bW9kZWxfbmFtZSwKICAgICAgICBtYXhfbmV3X3Rva2Vucz1tYXhfbmV3X3Rva2VucywKICAgICAgICBjaHVua19sZW5ndGhfcz1jaHVua19sZW5ndGhfcywKICAgICAgICBiYXRjaF9zaXplPWJhdGNoX3NpemUsCiAgICAgICAgcmV0dXJuX3RpbWVzdGFtcHM9KAogICAgICAgICAgICAid29yZCIKICAgICAgICAgICAgaWYgc3BlZWNoX2RpYXJpemF0aW9uIGlzIG5vdCBOb25lIG9yIHNwZWVjaF9kaWFyaXplX3Blcl9jaGFubmVsIGlzIG5vdCBOb25lCiAgICAgICAgICAgIGVsc2UgRmFsc2UKICAgICAgICApLAogICAgICAgIHBlcl9jaGFubmVsX3RyYW5zY3JpcHRpb249c3BlZWNoX2RpYXJpemVfcGVyX2NoYW5uZWwgb3IgMCwKICAgICAgICBzcG9rZW5fbGFuZ3VhZ2U9c3Bva2VuX2xhbmd1YWdlLAogICAgICAgIHRyYW5zbGF0ZV90b19lbmdsaXNoPXRyYW5zbGF0ZV90b19lbmdsaXNoLAogICAgKQoKICAgICMgUnVuIHRoZSB0cmFuc2NyaXB0aW9uOgogICAgaWYgdXNlX211bHRpcHJvY2Vzc2luZzoKICAgICAgICByZXN1bHRzID0gX3BhcmFsbGVsX3J1bigKICAgICAgICAgICAgbl93b3JrZXJzPXVzZV9tdWx0aXByb2Nlc3NpbmcKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZSh1c2VfbXVsdGlwcm9jZXNzaW5nLCBpbnQpCiAgICAgICAgICAgIGVsc2UgMSwKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGJhdGNoX3Byb2Nlc3Nvcj1iYXRjaF9wcm9jZXNzb3IsCiAgICAgICAgICAgIHRyYW5zY3JpYmVyPXRyYW5zY3JpYmVyLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGJhdGNoX3Byb2Nlc3Nvcj1iYXRjaF9wcm9jZXNzb3IsCiAgICAgICAgICAgIHRyYW5zY3JpYmVyPXRyYW5zY3JpYmVyLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgICMgUHJvY2VzcyB0aGUgcmVzdWx0czoKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJTdW1tYXJpemluZyB0aGUgcmVzdWx0cy4iKQogICAgc3VjY2Vzc2VzID0gW10KICAgIGVycm9ycyA9IHt9CiAgICBmb3IgaXNfZXJyb3IsIHJlc3VsdCBpbiByZXN1bHRzOgogICAgICAgIGlmIGlzX2Vycm9yOgogICAgICAgICAgICBlcnJvcnNbcmVzdWx0WzBdXSA9IHJlc3VsdFsxXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHN1Y2Nlc3Nlcy5hcHBlbmQocmVzdWx0KQogICAgc3VjY2Vzc2VzID0gcGQuRGF0YUZyYW1lKHN1Y2Nlc3NlcywgY29sdW1ucz1bImF1ZGlvX2ZpbGUiLCAidHJhbnNjcmlwdGlvbl9maWxlIl0pCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygKICAgICAgICAgICAgZiJEb25lICh7c3VjY2Vzc2VzLnNoYXBlWzBdfS97bGVuKGF1ZGlvX2ZpbGVzKX0pXG4iCiAgICAgICAgICAgIGYiVHJhbnNjcmlwdGlvbnMgc3VtbWFyeTpcbiIKICAgICAgICAgICAgZiJ7c3VjY2Vzc2VzLmhlYWQoKX0iCiAgICAgICAgKQoKICAgIHJldHVybiBzdHIob3V0cHV0X2RpcmVjdG9yeSksIHN1Y2Nlc3NlcywgZXJyb3JzCgoKZGVmIF9nZXRfYXVkaW9fZmlsZXMoCiAgICBkYXRhX3BhdGg6IFVuaW9uW1BhdGgsIHN0ciwgbGlzdF0sCikgLT4gTGlzdFtQYXRoXToKICAgICIiIgogICAgR2V0IHRoZSBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlLiBJZiBhIHBhdGggdG8gYSBkaXJlY3RvcnkgaXMgZ2l2ZW4sIGFsbCBmaWxlcyBpbiB0aGUgZGlyZWN0b3J5IHdpbGwgYmUgY29sbGVjdGVkLgoKICAgIDpwYXJhbSBkYXRhX3BhdGg6IFRoZSBkYXRhIHBhdGggdG8gY29sbGVjdCB0aGUgYXVkaW8gZmlsZXMgZnJvbS4KCiAgICA6cmV0dXJuczogVGhlIGF1ZGlvIGZpbGVzIGxpc3QuCiAgICAiIiIKICAgICMgQ2hlY2sgaWYgZ2l2ZW4gYSBsaXN0IG9mIHBhdGhzOgogICAgaWYgaXNpbnN0YW5jZShkYXRhX3BhdGgsIGxpc3QpOgogICAgICAgIGF1ZGlvX2ZpbGVzID0gW10KICAgICAgICBmb3IgcGF0aCBpbiBkYXRhX3BhdGg6CiAgICAgICAgICAgIGF1ZGlvX2ZpbGVzLmV4dGVuZChfZ2V0X2F1ZGlvX2ZpbGVzKGRhdGFfcGF0aD1wYXRoKSkKICAgICAgICByZXR1cm4gYXVkaW9fZmlsZXMKCiAgICAjIENoZWNrIGlmIGdpdmVuIGEgc2luZ2xlIHN0cmluZyBwYXRoIHRvIGNhc3QgaXQgdG8gYSBgcGF0aGxpYi5QYXRoYDoKICAgIGlmIGlzaW5zdGFuY2UoZGF0YV9wYXRoLCBzdHIpOgogICAgICAgIGRhdGFfcGF0aCA9IFBhdGgoZGF0YV9wYXRoKS5hYnNvbHV0ZSgpCgogICAgIyBDaGVjayBpZiB0aGUgcGF0aCBpcyBvZiBhIGRpcmVjdG9yeSBvciBhIGZpbGU6CiAgICBpZiBkYXRhX3BhdGguaXNfZGlyKCk6CiAgICAgICAgIyBHZXQgYWxsIGZpbGVzIGluc2lkZSB0aGUgZGlyZWN0b3J5OgogICAgICAgIGF1ZGlvX2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgYXVkaW9fZmlsZXMgPSBbZGF0YV9wYXRoXQogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlVucmVjb2duaXplZCBkYXRhIHBhdGguIFRoZSBwYXJhbWV0ZXIgYGRhdGFfcGF0aGAgbXVzdCBiZSBhIHZhbGlkIHBhdGggdG8gZWl0aGVyIGEgZGlyZWN0b3J5IHBhdGggb3IgYSAiCiAgICAgICAgICAgIGYiZmlsZS4gR2l2ZW46IHtzdHIoZGF0YV9wYXRoKX0gIgogICAgICAgICkKCiAgICByZXR1cm4gYXVkaW9fZmlsZXMKCgpkZWYgX3J1bigKICAgIGF1ZGlvX2ZpbGVzOiBMaXN0W1BhdGhdLAogICAgYmF0Y2hfcHJvY2Vzc29yOiBCYXRjaFByb2Nlc3NvciwKICAgIHRyYW5zY3JpYmVyOiBUcmFuc2NyaWJlciwKICAgIHZlcmJvc2U6IGJvb2wsCikgLT4gTGlzdFtUdXBsZVtib29sLCBUdXBsZVtzdHIsIHN0cl1dXToKICAgICIiIgogICAgUnVuIHRoZSB0cmFuc2NyaXB0aW9uIHdpdGhvdXQgbXVsdGlwcm9jZXNzaW5nLgoKICAgIDpwYXJhbSBhdWRpb19maWxlczogICAgIFRoZSBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlLgogICAgOnBhcmFtIGJhdGNoX3Byb2Nlc3NvcjogVGhlIGJhdGNoIHByb2Nlc3NvciB0byB1c2UuCiAgICA6cGFyYW0gdHJhbnNjcmliZXI6ICAgICBUaGUgdHJhbnNjcmliZXIgdG8gdXNlLgogICAgOnBhcmFtIHZlcmJvc2U6ICAgICAgICAgVmVyYm9zaXR5LgoKICAgIDpyZXR1cm5zOiBUaGUgY29sbGVjdGVkIHJlc3VsdHMuCiAgICAiIiIKICAgICMgTG9hZCB0aGUgdHJhbnNjcmlwdGlvbiBwaXBlbGluZToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiTG9hZGluZyB0aGUgdHJhbnNjcmlwdGlvbiBwaXBlbGluZS4iKQogICAgdHJhbnNjcmliZXIubG9hZCgpCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiVHJhbnNjcmlwdGlvbiBwaXBlbGluZSBsb2FkZWQuIikKCiAgICAjIFRyYW5zY3JpYmUgdGhlIGZpbGVzOgogICAgdHJhbnNjcmliZXIudHJhbnNjcmliZSgKICAgICAgICBhdWRpb19maWxlcz1hdWRpb19maWxlcywKICAgICAgICBiYXRjaF9wcm9jZXNzb3I9YmF0Y2hfcHJvY2Vzc29yLAogICAgICAgIHZlcmJvc2U9dmVyYm9zZSwKICAgICkKCiAgICAjIFJldHVybiB0aGUgcmVzdWx0czoKICAgIHJldHVybiBiYXRjaF9wcm9jZXNzb3IuZ2V0X3Jlc3VsdHMoKQoKCmRlZiBfcGFyYWxsZWxfcnVuKAogICAgbl93b3JrZXJzOiBpbnQsCiAgICBhdWRpb19maWxlczogTGlzdFtQYXRoXSwKICAgIGJhdGNoX3Byb2Nlc3NvcjogQmF0Y2hQcm9jZXNzb3IsCiAgICB0cmFuc2NyaWJlcjogVHJhbnNjcmliZXIsCiAgICB2ZXJib3NlOiBib29sLAopOgogICAgIiIiCiAgICBSdW4gdGhlIHRyYW5zY3JpcHRpb24gd2l0aCBtdWx0aXByb2Nlc3NpbmcuCgogICAgOnBhcmFtIG5fd29ya2VyczogICAgICAgVGhlIGFtb3VudCBvZiB3b3JrZXJzIHRvIHVzZSBhcyB0YXNrIGNvbXBsZXRlcnMuCiAgICA6cGFyYW0gYXVkaW9fZmlsZXM6ICAgICBUaGUgYXVkaW8gZmlsZXMgdG8gdHJhbnNjcmliZS4KICAgIDpwYXJhbSBiYXRjaF9wcm9jZXNzb3I6IFRoZSBiYXRjaCBwcm9jZXNzb3IgdG8gdXNlLgogICAgOnBhcmFtIHRyYW5zY3JpYmVyOiAgICAgVGhlIHRyYW5zY3JpYmVyIHRvIHVzZS4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgIFZlcmJvc2l0eS4KCiAgICA6cmV0dXJuczogVGhlIGNvbGxlY3RlZCByZXN1bHRzLgogICAgIiIiCiAgICAjIEluaXRpYWxpemUgdGhlIG11bHRpcHJvY2Vzc2luZyBxdWV1ZXM6CiAgICBiYXRjaGVzX3F1ZXVlID0gUXVldWUoKQogICAgdGFza3NfcXVldWUgPSBRdWV1ZSgpCiAgICByZXN1bHRzX3F1ZXVlID0gUXVldWUoKQoKICAgICMgSW5pdGlhbGl6ZSB0aGUgbXVsdGlwcm9jZXNzaW5nIHByb2Nlc3NlczoKICAgIGJhdGNoX3Byb2Nlc3NpbmdfcHJvY2VzcyA9IFByb2Nlc3MoCiAgICAgICAgdGFyZ2V0PV9tdWx0aXByb2Nlc3NpbmdfcHJvY2Vzc19iYXRjaGVzLAogICAgICAgIGt3YXJncz17CiAgICAgICAgICAgICJiYXRjaF9wcm9jZXNzb3IiOiBiYXRjaF9wcm9jZXNzb3IsCiAgICAgICAgICAgICJiYXRjaGVzX3F1ZXVlIjogYmF0Y2hlc19xdWV1ZSwKICAgICAgICAgICAgInRhc2tzX3F1ZXVlIjogdGFza3NfcXVldWUsCiAgICAgICAgICAgICJuX3Rhc2tfY29tcGxldGVycyI6IG5fd29ya2VycywKICAgICAgICB9LAogICAgKQogICAgdGFza19jb21wbGV0aW9uX3Byb2Nlc3NlcyA9IFsKICAgICAgICBQcm9jZXNzKAogICAgICAgICAgICB0YXJnZXQ9X211bHRpcHJvY2Vzc2luZ19jb21wbGV0ZV90YXNrcywKICAgICAgICAgICAga3dhcmdzPXsidGFza3NfcXVldWUiOiB0YXNrc19xdWV1ZSwgInJlc3VsdHNfcXVldWUiOiByZXN1bHRzX3F1ZXVlfSwKICAgICAgICApCiAgICAgICAgZm9yIF8gaW4gcmFuZ2Uobl93b3JrZXJzKQogICAgXQoKICAgICMgU3RhcnQgdGhlIG11bHRpcHJvY2Vzc2luZyBwcm9jZXNzZXM6CiAgICBiYXRjaF9wcm9jZXNzaW5nX3Byb2Nlc3Muc3RhcnQoKQogICAgZm9yIHAgaW4gdGFza19jb21wbGV0aW9uX3Byb2Nlc3NlczoKICAgICAgICBwLnN0YXJ0KCkKCiAgICAjIExvYWQgdGhlIHRyYW5zY3JpcHRpb24gcGlwZWxpbmU6CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkxvYWRpbmcgdGhlIHRyYW5zY3JpcHRpb24gcGlwZWxpbmUuIikKICAgIHRyYW5zY3JpYmVyLmxvYWQoKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIlRyYW5zY3JpcHRpb24gcGlwZWxpbmUgbG9hZGVkLiIpCgogICAgIyBUcmFuc2NyaWJlIHRoZSBmaWxlczoKICAgIHRyYW5zY3JpYmVyLnRyYW5zY3JpYmUoCiAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsIGJhdGNoZXNfcXVldWU9YmF0Y2hlc19xdWV1ZSwgdmVyYm9zZT12ZXJib3NlCiAgICApCgogICAgIyBDb2xsZWN0IHRoZSByZXN1bHRzOgogICAgcmVzdWx0cyA9IFtdCiAgICBzdG9wX21hcmtzX2NvdW50ZXIgPSAwCiAgICB3aGlsZSBUcnVlOgogICAgICAgICMgR2V0IGEgcmVzdWx0IGZyb20gdGhlIHF1ZXVlOgogICAgICAgIHJlc3VsdDogVHVwbGVbYm9vbCwgVHVwbGVbc3RyLCBzdHJdXSA9IHJlc3VsdHNfcXVldWUuZ2V0KCkKICAgICAgICBpZiByZXN1bHQgPT0gX01VTFRJUFJPQ0VTU0lOR19TVE9QX01BUks6CiAgICAgICAgICAgIHN0b3BfbWFya3NfY291bnRlciArPSAxCiAgICAgICAgICAgIGlmIHN0b3BfbWFya3NfY291bnRlciA9PSBuX3dvcmtlcnM6CiAgICAgICAgICAgICAgICBicmVhawogICAgICAgIGVsc2U6CiAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgcmVzdWx0OgogICAgICAgICAgICByZXN1bHRzLmFwcGVuZChyZXN1bHQpCgogICAgIyBXYWl0IGZvciB0aGUgcHJvY2Vzc2VzIHRvIGZpbmlzaDoKICAgIHJlc3VsdHNfcXVldWUuZW1wdHkoKQogICAgYmF0Y2hfcHJvY2Vzc2luZ19wcm9jZXNzLmpvaW4oKQogICAgZm9yIHAgaW4gdGFza19jb21wbGV0aW9uX3Byb2Nlc3NlczoKICAgICAgICBwLmpvaW4oKQoKICAgIHJldHVybiByZXN1bHRz base_image: mlrun/mlrun commands: [] code_origin: '' origin_filename: '' requirements: - - openai-whisper + - transformers - tqdm + - torchaudio + - torch entry_points: - open_mpi_handler: - name: open_mpi_handler - doc: '' + do_task: + name: do_task + doc: Try to perform the task storing an error if occurred. parameters: - - name: worker_inputs - type: List[str] - - name: root_worker_inputs - type: Dict[str, Any] - default: null + - name: self + outputs: [] + lineno: 348 + has_varargs: false + has_kwargs: false + is_failed: + name: is_failed + doc: Check if the task failed. + parameters: + - name: self outputs: - - default: '' - lineno: 29 - decorator: - name: decorator - doc: '' + - doc: Whether the task failed. + type: bool + lineno: 70 + has_varargs: false + has_kwargs: false + get_result: + name: get_result + doc: 'Get the result of the task. If the task failed, the error will be returned, + otherwise, the result will be the + + text file name.' parameters: - - name: handler + - name: self outputs: - - default: '' - lineno: 41 - wrapper: - name: wrapper - doc: '' - parameters: [] + - doc: The task's result. + type: Tuple[str, str] + lineno: 78 + has_varargs: false + has_kwargs: false + to_tuple: + name: to_tuple + doc: Convert the task to a tuple to reconstruct it later (used for multiprocessing + to pass in queue). + parameters: + - name: self + outputs: + - doc: The converted task. + type: Tuple[str, dict] + lineno: 358 + has_varargs: false + has_kwargs: false + transcription_output_channels: + name: transcription_output_channels + doc: Get the transcription output channels. + parameters: + - name: self + outputs: + - doc: The transcription output channels. + type: List[Tuple[str, dict]] + lineno: 340 + has_varargs: false + has_kwargs: false + process_batch: + name: process_batch + doc: 'Process a batch of transcriptions. Tasks related to the given batch will + be created and stored in the batch + + processor.' + parameters: + - name: self + - name: batch + type: List[dict] + doc: The batch of transcriptions to process. + outputs: [] + lineno: 575 + has_varargs: false + has_kwargs: false + get_tasks: + name: get_tasks + doc: Get the tasks to perform. + parameters: + - name: self + outputs: + - doc: The tasks to perform. + type: List[BaseTask] + lineno: 453 + has_varargs: false + has_kwargs: false + do_tasks: + name: do_tasks + doc: Perform the tasks. Should be used if no multiprocessing queue is given + to a transcriber. + parameters: + - name: self + outputs: [] + lineno: 463 + has_varargs: false + has_kwargs: false + get_results: + name: get_results + doc: Get the results of the tasks. The stored results are then cleared. + parameters: + - name: self outputs: - - default: '' - lineno: 46 + - doc: The results of the tasks. + type: List[Tuple[bool, Tuple[str, str]]] + lineno: 471 + has_varargs: false + has_kwargs: false + load: + name: load + doc: Load the transcriber. Must be called before transcribing. + parameters: + - name: self + outputs: [] + lineno: 695 + has_varargs: false + has_kwargs: false transcribe: name: transcribe - doc: 'Transcribe audio files into text files and collect additional data. The - end result is a directory of transcribed - - text files and a dataframe containing the following columns: - - - * audio_file - The audio file path. - - * transcription_file - The transcribed text file name in the output directory. - - * language - The detected language in the audio file. - - * language_probability - The detected language probability. - - * duration - The duration (in seconds) of the audio file (only if `audio_duration` - is set to True).' + doc: "Transcribe audio files into text files and collect additional data. The\ + \ end result is a directory of transcribed\ntext files and a dataframe containing\ + \ the following columns:\n\n* audio_file - The audio file path.\n* transcription_file\ + \ - The transcribed text file name in the output directory.\n\nThe transcription\ + \ is based on Huggingface's ASR pipeline -\nhttps://huggingface.co/transformers/main_classes/pipelines.html#transformers.AutomaticSpeechRecognitionPipeline\ + \ and\nis tested with OpenAI's Whisper models - https://huggingface.co/openai.\n\ + \nIf one of the speaker diarization parameters are given (either `speech_diarization`\ + \ or\n`speech_diarize_per_channel`), the transcription will be written in\ + \ a conversation format, where each speaker will\nbe written in a separate\ + \ line::\n\n speaker_1: text\n speaker_2: text\n speaker_1: text\n\ + \ ..." parameters: - name: data_path - type: Union[str, List[str]] + type: Union[str, Path, List[Union[str, Path]]] doc: A directory of audio files or a single file or a list of files to transcribe. - name: output_directory type: str - doc: Path to a directory to save all transcribed audio files. + doc: Path to a directory to save all transcribed audio files. If not given, + will save the transcribed files in a temporary directory. + default: null - name: model_name type: str - doc: 'One of the official model names of Whisper: {''tiny.en'', ''tiny'', - ''base.en'', ''base'', ''small.en'', ''small'', ''medium.en'', ''medium'', - ''large-v1'', ''large-v2'', ''large''} or a full name of a fine-tuned whisper - model from the huggingface hub.' - default: base + doc: 'The model name to use. Should be a model from the OpenAI''s Whisper + models for best results (for example "tiny", "base", "large", etc.). See + here for more information: https://huggingface.co/openai?search_models=whisper.' + default: openai/whisper-tiny - name: device - type: Literal[, , ] - doc: Device to load the model. Can be one of {"cuda", "cpu"}. Default will - prefer "cuda" if available. To use a specific GPU or more than one GPU, - pass the `device_index` argument via the `init_kwargs`. - default: auto - - name: compute_type type: str - doc: 'The data type to use for computation. For more information, check https://opennmt.net/CTranslate2/quantization.html. - Default: "default" - will use the default type depending on the device used.' - default: default - - name: language + doc: The device to use for inference. If not given, will use GPU if available. + default: null + - name: use_flash_attention_2 + type: bool + doc: 'Whether to use the Flash Attention 2 implementation. It can be used + only with one of the following GPUs: Nvidia H series and Nvidia A series. + T4 support will be available soon.' + default: null + - name: use_better_transformers + type: bool + doc: Whether to use the Better Transformers library to further optimize the + model. Should be used for all use cases that do not support flash attention + 2. + default: null + - name: assistant_model + type: str + doc: 'The assistant model name to use for inference. Notice that the optimizations + (flash attention 2 and better transformers) will be applied for the assistant + as well. Should be a model from Huggingface''s distil-whisper (see here + for more information: https://github.com/huggingface/distil-whisper).' + default: null + - name: max_new_tokens + type: int + doc: The maximum number of new tokens to generate. This is used to limit the + generation length. Default is 128 tokens. + default: 128 + - name: chunk_length_s + type: int + doc: The audio chunk to split the audio to (in seconds). Default is 30 seconds. + default: 30 + - name: batch_size + type: int + doc: The batch size to use for inference. Default is 2. + default: 8 + - name: spoken_language type: str - doc: 'The spoken language to force Whisper the output language. If None, the - Whisper model will automatically predict the output langauge. Default: None.' + doc: Aim whisper to know what language is spoken. If None, it will try to + detect it. default: null - name: translate_to_english type: bool - doc: 'Whether to translate the English post transcription. Default: False.' + doc: Whether to translate the transcriptions to English. default: false - name: speech_diarization type: Dict[str, List[Tuple[float, float, str]]] doc: 'A speech diarization dictionary with the file names to transcribe as keys and their diarization as value. The diarization is a list of tuples: - (start, end, speaker). The transcription result will be in the following - format: "{speaker}: text text text.". Files with missing diarizations will - print a warning. Pay attention the diarization must be for the entire duration - of the audio file (as long as Whisper is predicting words up until then).' + (start, end, speaker). An example for a diarization dictionary::' default: null - - name: audio_duration - type: bool - doc: 'Whether to include the audio files duration (in seconds). The estimated - duration is from bitrate and may be inaccurate. Default: False.' - default: false - - name: init_kwargs - type: dict - doc: Additional `WhisperModel.__init__` keyword arguments to use. + - name: speech_diarize_per_channel + type: int + doc: 'Perform speech diarization per channel. Each speaker is expected to + belong to a separate channel in the audio. Notice: This will make the transcription + slower as each channel wil be transcribed separatly. If a speech diarization + is passed (via the `speech_diarization` parameter), this parameter is ignored.' default: null - - name: transcribe_kwargs - type: dict - doc: Additional `WhisperModel.transcribe` keyword arguments to use. + - name: speaker_labels + type: List[str] + doc: A list of speaker labels by channel order to use for writing the transcription + with respect to per channel speech diarization. This won't be used together + with a given speech diarization (via the `speech_diarization` parameter). default: null + - name: use_multiprocessing + type: Union[bool, int] + doc: 'Whether to use multiprocessing to transcribe the audio files. Can be + either a boolean value or an integer. If `True`, will use the default amount + of workers (3): 1 for transcription, 1 for batch processing and 1 for task + completion (such as speech diarization and writing to files). To control + the amount of tasks completion workers, an integer can be provided to specify + the amount of workers. `False`, will use a single process. Default is `False`.' + default: false - name: verbose type: bool - doc: 'Whether to present logs of a progress bar and errors. Default: False.' + doc: Whether to print the progress of the transcription. Default is `False`. default: false + outputs: [] + lineno: 1097 + has_varargs: false + has_kwargs: false + audio_iterator: + name: audio_iterator + doc: '' + parameters: [] + outputs: + - type: Generator[Union[dict, str], None, None] + lineno: 804 + has_varargs: false + has_kwargs: false + batch_iterator: + name: batch_iterator + doc: '' + parameters: [] outputs: - - doc: 'A tuple of:' - default: '' - lineno: 135 + - type: Generator[List[Union[dict, str]], None, None] + lineno: 816 + has_varargs: false + has_kwargs: false + open_mpi_handler: + name: open_mpi_handler + doc: '' + parameters: + - name: worker_inputs + type: List[str] + - name: root_worker_inputs + type: Dict[str, Any] + default: null + outputs: [] + lineno: 957 + has_varargs: false + has_kwargs: false + decorator: + name: decorator + doc: '' + parameters: + - name: handler + outputs: [] + lineno: 969 + has_varargs: false + has_kwargs: false + wrapper: + name: wrapper + doc: '' + parameters: [] + outputs: [] + lineno: 974 + has_varargs: false + has_kwargs: true description: Transcribe audio files into text files default_handler: transcribe disable_auto_mount: false diff --git a/transcribe/item.yaml b/transcribe/item.yaml index 28bc5a1c0..d53341ff2 100644 --- a/transcribe/item.yaml +++ b/transcribe/item.yaml @@ -21,8 +21,10 @@ spec: image: mlrun/mlrun kind: job requirements: - - openai-whisper + - transformers - tqdm + - torchaudio + - torch + - accelerate url: '' -version: 0.0.2 -test_valid: True +version: 1.0.0 \ No newline at end of file diff --git a/transcribe/requirements.txt b/transcribe/requirements.txt index 47af1e515..d16bfc9dd 100644 --- a/transcribe/requirements.txt +++ b/transcribe/requirements.txt @@ -1,3 +1,5 @@ -faster-whisper +transformers +torch +torchaudio tqdm -librosa \ No newline at end of file +accelerate \ No newline at end of file diff --git a/transcribe/test_transcribe.py b/transcribe/test_transcribe.py index 9f89cddbb..f70b3856d 100644 --- a/transcribe/test_transcribe.py +++ b/transcribe/test_transcribe.py @@ -14,13 +14,13 @@ # import os import pathlib -import sys import tempfile from difflib import SequenceMatcher import mlrun import pytest + expected_outputs = [ "This is a speech to text test.", "In the heart of the stadium, " @@ -29,24 +29,21 @@ "The crowd roars, a symphony of passion, " "as the game writes its unpredictable story on the field of destiny.", ] -whisper_models = [ - "tiny.en", - "tiny", - "base.en", - "base", +models = [ + + "openai/whisper-tiny", ] -@pytest.mark.skipif( - condition=sys.version_info[:2] < (3, 8), - reason="whisper requires python 3.8 and above" -) -@pytest.mark.parametrize("model_name", whisper_models) +@pytest.mark.skipif(os.system("which ffmpeg") != 0, reason="ffmpeg not installed") +@pytest.mark.parametrize("model_name", models) @pytest.mark.parametrize("audio_path", ["./data", "./data/speech_01.mp3"]) def test_transcribe(model_name: str, audio_path: str): # Setting variables and importing function: artifact_path = tempfile.mkdtemp() - transcribe_function = mlrun.import_function("function.yaml") + project = mlrun.get_or_create_project("test") + transcribe_function = project.set_function("transcribe.py", "transcribe", kind="job", image="mlrun/mlrun") + # transcribe_function = mlrun.import_function("function.yaml") temp_dir = tempfile.mkdtemp() # Running transcribe function: @@ -56,7 +53,6 @@ def test_transcribe(model_name: str, audio_path: str): "data_path": audio_path, "model_name": model_name, "device": "cpu", - "compute_type": "int8", "output_directory": temp_dir, }, local=True, diff --git a/transcribe/transcribe.py b/transcribe/transcribe.py index bcd37f5c5..9cabcb1e8 100644 --- a/transcribe/transcribe.py +++ b/transcribe/transcribe.py @@ -1,4 +1,4 @@ -# Copyright 2023 Iguazio +# Copyright 2024 Iguazio # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,16 +11,944 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging import operator -import pathlib +import os +import tempfile from functools import reduce, wraps -from typing import Any, Dict, List, Literal, NamedTuple, Tuple, Union +from multiprocessing import Process, Queue +from pathlib import Path +from typing import Any, Dict, Generator, List, Literal, NamedTuple, Tuple, Union -import faster_whisper import pandas as pd +import torch +import torchaudio from tqdm import tqdm +from transformers import ( + AutomaticSpeechRecognitionPipeline, + AutoModelForCausalLM, + pipeline, +) +from transformers.utils import is_flash_attn_2_available + + +class BaseTask: + """ + A task to write the transcription to file. + """ + + def __init__( + self, audio_file: Path, transcription_output: Union[dict, str], text_file: Path + ): + """ + Initialize the task. + + :param audio_file: Path to the audio file that was transcribed. + :param transcription_output: The transcription output from the pipeline. String means an exception was raised. + :param text_file: Path to the text file to write the transcription to. + """ + # Store the parameters: + self._audio_file = audio_file + self._transcription_output = transcription_output + self._text_file = text_file + + # Prepare the error variable: + self._error: str = None + + def do_task(self): + """ + Try to perform the task storing an error if occurred. + """ + if isinstance(self._transcription_output, str): + self._error = self._transcription_output + return + try: + self._do_task() + except Exception as exception: + self._error = str(exception) + + def is_failed(self) -> bool: + """ + Check if the task failed. + + :returns: Whether the task failed. + """ + return self._error is not None + + def get_result(self) -> Tuple[str, str]: + """ + Get the result of the task. If the task failed, the error will be returned, otherwise, the result will be the + text file name. + + :returns: The task's result. + """ + if self.is_failed(): + return self._audio_file.name, self._error + return self._audio_file.name, self._text_file.name + + def to_tuple(self) -> Tuple[str, dict]: + """ + Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). + + :returns: The converted task. + """ + return self.__class__.__name__, { + "audio_file": self._audio_file, + "transcription_output": self._transcription_output, + "text_file": self._text_file, + } + + def _do_task(self): + """ + Perform the task - write the transcription to the stored file path. + """ + # Checking for no duplications: + i = 1 + while self._text_file.exists(): + i += 1 + self._text_file = ( + self._text_file.parent + / f"{self._text_file.stem.rsplit('_', 1)[0]}_{i}{self._text_file.suffix}" + ) + + # Make sure all directories are created: + self._text_file.parent.mkdir(exist_ok=True, parents=True) + + # Write to file: + with open(self._text_file, "w") as fp: + fp.write(self._transcription_output["text"]) + + +class SpeechDiarizationTask(BaseTask): + """ + A task to write the transcription to file with respect to a given speech diarization. + """ + + class _DiarizationSegment(NamedTuple): + """ + A speech diarization segment. + """ + + start: float + end: float + speaker: str + + class _WordTimestamp(NamedTuple): + """ + A word with its start and end timestamps. + """ + + start: float + end: float + text: str + + def __init__( + self, + audio_file: Path, + transcription_output: dict, + text_file: Path, + speech_diarization: List[Tuple[float, float, str]], + ): + """ + Initialize the task. + + :param audio_file: Path to the audio file that was transcribed. + :param transcription_output: The transcription output from the pipeline. + :param text_file: Path to the text file to write the transcription to. + :param speech_diarization: A speech diarization as a list of tuples: (start, end, speaker). + """ + super().__init__( + audio_file=audio_file, + transcription_output=transcription_output, + text_file=text_file, + ) + self._speech_diarization = speech_diarization + self._segments: List[SpeechDiarizationTask._DiarizationSegment] = None + self._last_chosen_index = 0 + + def to_tuple(self) -> Tuple[str, dict]: + """ + Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). + + :returns: The converted task. + """ + task_class, task_kwargs = super().to_tuple() + return task_class, { + **task_kwargs, + "speech_diarization": self._speech_diarization, + } + + def _do_task(self): + """ + Perform the task - write the transcription to the stored file path with respect to the given speech diarization. + """ + # Check if a speech diarization is given, if not, just write the transcription to file: + if not self._speech_diarization: + super()._do_task() + return + + # Cast the chunks to word timestamps tuples: + words = [ + SpeechDiarizationTask._WordTimestamp( + start=chunk["timestamp"][0], + end=chunk["timestamp"][1], + text=chunk["text"], + ) + for chunk in self._transcription_output["chunks"] + ] + + # Cast speech diarization to segments tuples: + self._segments = [ + SpeechDiarizationTask._DiarizationSegment(*segment) + for segment in self._speech_diarization + ] + + # Try to match the Whisper model predicted timestamps to the closest diarization segment (closest diarization + # segment will be the most overlapping with the word, and if there is no overlap, the closest segment to the + # word): + speaker = self._segments[self._last_chosen_index].speaker + text = f"{speaker}:" + for word in words: + # Get the next diarization segment: + self._get_next_segment(word=word) + # Check if the segment is of the same speaker: + if self._segments[self._last_chosen_index].speaker == speaker: + # Collect the word: + text += word.text + else: + # Append a newline and update the new speaker: + speaker = self._segments[self._last_chosen_index].speaker + text += f"\n{speaker}:{word.text}" + + # Update the transcription output with the new text to write it to file: + self._transcription_output["text"] = text + super()._do_task() + + def _get_next_segment( + self, + word: _WordTimestamp, + ): + """ + Get the next diarization segment the given word falls into. The `self._last_chosen_index` will be updated + accordingly. + + :param word: The word timestamp to match to the next segment. + """ + # If the last chosen segment is the last segment, return it: + if self._last_chosen_index == len(self._segments) - 1: + return + + # Get the last chosen diarization segment: + last_chosen = self._segments[self._last_chosen_index] + + # None value may appear if the word is the last word in the audio file, or it was split during inference. In + # that case, we'll set the last segment: + if word.end is None: + self._last_chosen_index = len(self._segments) - 1 + return + + # If the word ends before the last chosen segment: + if word.end <= last_chosen.start: + # Then it is still the closest segment + return + + # We check if it ends inside the last chosen segment: + if word.end < last_chosen.end: + # Then it still is the closest segment + return + + # The word ends after the segment, we need to collect all next segments up until the word ends before them: + possible_segments = [self._last_chosen_index] + for i in range(self._last_chosen_index + 1, len(self._segments)): + if word.end > self._segments[i].end: + possible_segments.append(i) + continue + possible_segments.append(i) + break + + # Check for the most overlapping option: + best_overlap = 0 + most_overlapping_segment_index = None + for i in possible_segments: + # If the word starts before segment: + if word.start <= self._segments[i].start: + # If it ends before the segment, there is an overlap from the start of the segment to the end of the + # word: + if word.end < self._segments[i].end: + overlap = word.end - self._segments[i].start + else: + # The word is wrapping the segment, the overlap is the segment's length: + overlap = self._segments[i].end - self._segments[i].start + # The word starts in segment, check if the word ends in it: + elif word.end < self._segments[i].end: + # The overlap is the word's length: + overlap = word.end - word.start + # The word start in segment but ends after it, the overlap is from the word's start to the segment's end: + else: + overlap = self._segments[i].end - word.start + # Check for new best overlap: + if overlap > best_overlap: + best_overlap = overlap + most_overlapping_segment_index = i + if most_overlapping_segment_index is not None: + self._last_chosen_index = most_overlapping_segment_index + return + + # If there is no overlapping segment, return the closest segment: + best_distance = None + closest_segment_index = None + for i in possible_segments: + distance = ( + word.start - self._segments[i].end + if word.start > self._segments[i].end + else self._segments[i].start - word.end + ) + if best_distance is None or distance < best_distance: + best_distance = distance + closest_segment_index = i + self._last_chosen_index = closest_segment_index + + +class SpeechDiarizationPerChannelTask(BaseTask): + """ + A task to write the transcription to file with respect to a given speech diarization per channel. + """ + + class _WordTimestamp(NamedTuple): + """ + A word with its start and end timestamps and speaker label (channel the word was taken from). + """ + + start: float + end: float + speaker: str + text: str + + def __init__(self, audio_file: Path, text_file: Path): + """ + Initialize the task. + + :param audio_file: Path to the audio file that was transcribed. + :param text_file: Path to the text file to write the transcription to. + """ + super().__init__( + audio_file=audio_file, transcription_output={}, text_file=text_file + ) + self._transcription_output_channels: List[Tuple[str, dict]] = [] + + @property + def transcription_output_channels(self) -> List[Tuple[str, dict]]: + """ + Get the transcription output channels. + + :returns: The transcription output channels. + """ + return self._transcription_output_channels + + def do_task(self): + """ + Try to perform the task storing an error if occurred. + """ + for _, channel_output in self._transcription_output_channels: + if isinstance(channel_output, str): + self._error = self._transcription_output_channels + return + super().do_task() + + def to_tuple(self) -> Tuple[str, dict]: + """ + Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). + + :returns: The converted task. + """ + task_class, task_kwargs = super().to_tuple() + task_kwargs.pop("transcription_output") + return task_class, task_kwargs + + def _do_task(self): + """ + Perform the task - write the transcription to the stored file path with respect to the given speech diarization + per channel. + """ + # Cast the chunks to word timestamps tuples: + words_per_channel = [ + [ + SpeechDiarizationPerChannelTask._WordTimestamp( + start=chunk["timestamp"][0], + end=chunk["timestamp"][1], + speaker=speaker, + text=chunk["text"], + ) + for chunk in output["chunks"] + ] + for speaker, output in self._transcription_output_channels + ] + + # Merge and sort the words per channel by their start time: + words = operator.add(*words_per_channel) + words.sort() + + # Write the transcription to file: + current_speaker = words[0].speaker + text = f"{current_speaker}:" + for word in words: + # Check if the word's speaker is different from the current one: + if word.speaker != current_speaker: + # Append a newline and update the new speaker: + current_speaker = word.speaker + text += f"\n{current_speaker}:" + # Collect the word: + text += word.text + + # Update the transcription output with the new text to write it to file: + self._transcription_output["text"] = text + super()._do_task() + + +class BatchProcessor: + """ + A batch processor to process batches of transcriptions. The batch processor is creating tasks and is aimed to be + working along the transcriber. It can be used with multiprocessing queue or run the tasks directly using the + associated methods. + """ + + def __init__(self, audio_files: List[Path], output_directory: Path): + """ + Initialize the batch processor. + + :param audio_files: The list of all audio files to transcribe. + :param output_directory: The output directory to write the transcriptions to. + """ + # Store the parameters: + self._audio_files = audio_files + self._output_directory = output_directory + + # Prepare the batching variables: + self._current_file_index = 0 + self._tasks: List[BaseTask] = [] + self._results: List[Tuple[bool, Tuple[str, str]]] = [] + + def process_batch(self, batch: List[Union[dict, str]]): + """ + Process a batch of transcriptions. Tasks related to the given batch will be created and stored in the batch + processor. + + :param batch: The batch of transcriptions to process. + """ + # Get the relevant files belongs to the given batch: + current_files = self._get_current_files(batch_size=len(batch)) + + # Build the diarization tasks: + self._tasks.extend( + [ + BaseTask( + audio_file=file, + transcription_output=batch[i], + text_file=self._output_directory / f"{file.stem}.txt", + ) + for i, file in enumerate(current_files) + ] + ) + + def get_tasks(self) -> List[BaseTask]: + """ + Get the tasks to perform. + + :returns: The tasks to perform. + """ + tasks = self._tasks + self._tasks = [] + return tasks + + def do_tasks(self): + """ + Perform the tasks. Should be used if no multiprocessing queue is given to a transcriber. + """ + for task in self.get_tasks(): + task.do_task() + self._results.append((task.is_failed(), task.get_result())) + + def get_results(self) -> List[Tuple[bool, Tuple[str, str]]]: + """ + Get the results of the tasks. The stored results are then cleared. + + :returns: The results of the tasks. + """ + results = self._results + self._results = [] + return results + + def _get_current_files(self, batch_size: int) -> List[Path]: + """ + Get the current files to process. + + :param batch_size: The batch size to progress the current file index. + + :returns: The current files to process. + """ + end_index = ( + self._current_file_index + batch_size + if self._current_file_index + batch_size < len(self._audio_files) + else len(self._audio_files) + ) + current_files = self._audio_files[self._current_file_index : end_index] + self._current_file_index = end_index + return current_files + + +class SpeechDiarizationBatchProcessor(BatchProcessor): + """ + A batch processor to process batches of transcriptions with respect to a given speech diarization. The batch + processor is creating tasks and is aimed to be working along the transcriber. It can be used with multiprocessing + queue or run the tasks directly using the associated methods. + """ + + def __init__( + self, audio_files: List[Path], output_directory: Path, speech_diarization: dict + ): + """ + Initialize the batch processor. + + :param audio_files: The list of all audio files to transcribe. + :param output_directory: The output directory to write the transcriptions to. + :param speech_diarization: A speech diarization dictionary to pass along with each processed batch. + """ + super().__init__(audio_files=audio_files, output_directory=output_directory) + self._speech_diarization = speech_diarization + self._audio_files = audio_files + + def process_batch(self, batch: List[dict]): + """ + Process a batch of transcriptions. Tasks related to the given batch will be created and stored in the batch + processor. + + :param batch: The batch of transcriptions to process. + """ + # Get the relevant files belongs to the given batch: + current_files = self._get_current_files(batch_size=len(batch)) + + # Build the diarization tasks: + self._tasks.extend( + [ + SpeechDiarizationTask( + audio_file=file, + transcription_output=batch[i], + text_file=self._output_directory / f"{file.stem}.txt", + speech_diarization=self._speech_diarization.get(file.name), + ) + for i, file in enumerate(current_files) + ] + ) + + +class PerChannelSpeechDiarizationBatchProcessor(BatchProcessor): + """ + A batch processor to process batches of transcriptions per channel. The batch processor is creating tasks with the + selected amount of channels given and is aimed to be working along the transcriber. It can be used with + multiprocessing queue or run the tasks directly using the associated methods. + """ + + def __init__( + self, + audio_files: List[Path], + output_directory: Path, + n_channels: int, + speakers: List[str], + ): + """ + Initialize the batch processor. + + :param audio_files: The list of all audio files to transcribe. + :param output_directory: The output directory to write the transcriptions to. + :param n_channels: The number of channels in each audio file to transcribe. + :param speakers: The speakers labels to use for each channel. + """ + super().__init__(audio_files=audio_files, output_directory=output_directory) + + # Store the parameters: + self._n_channels = n_channels + self._speakers = speakers + + # Prepare a channel buffer to store the channels until the current task created is fully covered: + self._task_in_process: SpeechDiarizationPerChannelTask = None + + def process_batch(self, batch: List[dict]): + """ + Process a batch of transcriptions. Tasks related to the given batch will be created and stored in the batch + processor. + + :param batch: The batch of transcriptions to process. + """ + # Go over the batch and create the tasks: + for output in batch: + # Check if there is a task in process: + if not self._task_in_process: + # Create a new task: + self._task_in_process = SpeechDiarizationPerChannelTask( + audio_file=self._audio_files[self._current_file_index], + text_file=self._output_directory + / f"{self._audio_files[self._current_file_index].stem}.txt", + ) + # Get the channel's speaker: + speaker = self._speakers[ + len(self._task_in_process.transcription_output_channels) + ] + # Collect the channel into the processed task: + self._task_in_process.transcription_output_channels.append( + (speaker, output) + ) + # Check if the task is fully covered (all channels are collected): + if ( + len(self._task_in_process.transcription_output_channels) + == self._n_channels + ): + # Collect the task and reset the task in process: + self._tasks.append(self._task_in_process) + self._current_file_index += 1 + self._task_in_process = None + + +class Transcriber: + """ + A transcription wrapper for the Huggingface's ASR pipeline - + https://huggingface.co/transformers/main_classes/pipelines.html#transformers.AutomaticSpeechRecognitionPipeline to + use with OpenAI's Whisper models - https://huggingface.co/openai. + """ + + def __init__( + self, + model_name: str, + device: str = None, + use_flash_attention_2: bool = None, + use_better_transformers: bool = None, + assistant_model: str = None, + max_new_tokens: int = 128, + chunk_length_s: int = 30, + batch_size: int = 2, + spoken_language: str = None, + translate_to_english: bool = False, + return_timestamps: Union[bool, Literal["word"]] = False, + per_channel_transcription: int = 0, + ): + """ + Initialize the transcriber. + + :param model_name: The model name to use. Should be a model from the OpenAI's Whisper models for + best results (for example "tiny", "base", "large", etc.). + :param device: The device to use for inference. If not given, will use GPU if available. + :param use_flash_attention_2: Whether to use the Flash Attention 2 implementation. It can be used only with + one of the following GPUs: Nvidia H series and Nvidia A series. T4 support + will be available soon. + + Note: If both `use_flash_attention_2` and + `use_better_transformers` are `None`, the optimization will be chosen + automatically according to the available resources. + + :param use_better_transformers: Whether to use the Better Transformers library to further optimize the model. + Should be used for all use cases that do not support flash attention 2. + + Note: If both `use_flash_attention_2` and `use_better_transformers` are + `None`, the optimization will be chosen automatically according to the + available resources. + :param assistant_model: The assistant model name to use for inference. Notice that the optimizations + (flash attention 2 and better transformers) will be applied for the assistant + as well. Should be a model from Huggingface's distil-whisper (see here for + more information: https://github.com/huggingface/distil-whisper). + :param max_new_tokens: The maximum number of new tokens to generate. This is used to limit the + generation length. Default is 128 tokens. + :param chunk_length_s: The audio chunk to split the audio to (in seconds). Default is 30 seconds. + :param batch_size: The batch size to use for inference. Default is 2. + :param spoken_language: Aim whisper to know what language is spoken. If None, it will try to detect it + for each chunk. + :param translate_to_english: Whether to translate the transcriptions to English. Default is False. + :param return_timestamps: Whether to return the timestamps of the words. If "word", will return the + timestamps of each word. If True will return the timestamps of each chunk. + Default is False. Aimed to be used for speech diarization. + :param per_channel_transcription: Whether to do per channel transcription. If needed to run per channel + transcription, pass the number of channels expected for each audio file here. + 0 means regular transcription (merge channels). + + Note: If `per_channel_transcription` is not 0, `batch_size` must be treated to + be the number of channels and not audio files. Aimed to be used for per + channel speech diarization. + """ + # Store loading parameters: + self._model_name = model_name + self._device = device + self._use_flash_attention_2 = use_flash_attention_2 + self._use_better_transformers = use_better_transformers + self._max_new_tokens = max_new_tokens + self._chunk_length_s = chunk_length_s + self._batch_size = batch_size + self._return_timestamps = return_timestamps + self._per_channel_transcription = per_channel_transcription + + # Store generation parameters: + self._assistant_model = assistant_model + self._spoken_language = spoken_language + self._translate_to_english = translate_to_english + + # Prepare the transcription objects: + self._transcription_pipeline: AutomaticSpeechRecognitionPipeline = None + self._generate_kwargs: dict = None + + def load(self): + """ + Load the transcriber. Must be called before transcribing. + """ + # Set the device and data type to use (prefer GPU if available): + device = torch.device( + self._device or "cuda" if torch.cuda.is_available() else "cpu" + ) + torch_dtype = torch.float16 if device.type == "cuda" else torch.float32 + + # Choose the optimization to use (in case the user did not specify any): + if ( + self._use_flash_attention_2 is None + and self._use_better_transformers is None + ): + # Prefer to use flash attention 2 if available and cuda device is supported (see GPU names to architecture + # here: https://en.wikipedia.org/wiki/List_of_Nvidia_graphics_processing_units#Tesla): + if device.type == "cuda" and is_flash_attn_2_available(): + cuda_device_name = torch.cuda.get_device_properties(device).name + if any( + cuda_device_name.startswith(gpu_name) + for gpu_name in [ + "NVIDIA A", # For Ampere architecture (e.g. A10, A30, A100) + "NVIDIA H", # For Hopper architecture (e.g. H100) + "NVIDIA L", # For Ada Lovelace architecture (e.g. L4, L40) + "NVIDIA RTX 30", # For Ada Lovelace architecture (RTX 30 series) + "NVIDIA RTX 40", # For Ada Lovelace architecture (RTX 40 series) + "NVIDIA RTX 50", # For Ada Lovelace architecture (RTX 50 series) + # Will be supported soon according to FlashAttention GitHub repo: + # https://github.com/Dao-AILab/flash-attention?tab=readme-ov-file#installation-and-features + # "NVIDIA T4", # For Turing architecture (only T4) + # "NVIDIA RTX 20", # For Turing architecture (RTX 20 series) + ] + ): + self._use_flash_attention_2 = True + else: + self._use_better_transformers = True + else: + self._use_better_transformers = True + + # Build the optimizations kwargs: + model_kwargs = { + "low_cpu_mem_usage": True, + "use_safetensors": True, + } + if self._use_flash_attention_2: + if _LOGGER: + _LOGGER.info( + "Using FlashAttention2 optimization - make sure the `flash-attn` package is installed via " + "`pip install -U flash-attn --no-build-isolation`" + ) + model_kwargs["attn_implementation"] = "flash_attention_2" + elif self._use_better_transformers: + if _LOGGER: + _LOGGER.info( + "Using BetterTransformers optimization - make sure the `optimum` package is installed via " + "`pip install -U optimum`" + ) + model_kwargs["attn_implementation"] = "sdpa" + + # Initialize the speech recognition pipeline: + self._transcription_pipeline = pipeline( + task="automatic-speech-recognition", + model=self._model_name, + model_kwargs=model_kwargs.copy(), + batch_size=self._batch_size, + max_new_tokens=self._max_new_tokens, + chunk_length_s=self._chunk_length_s, + return_timestamps=self._return_timestamps, + torch_dtype=torch_dtype, + device=device, + ) + + # Prepare the generation kwargs: + self._generate_kwargs = { + "language": self._spoken_language, + "task": "translate" if self._translate_to_english else "transcribe", + } + + # Initialize the assistant model (if needed): + if self._assistant_model: + assistant_model = AutoModelForCausalLM.from_pretrained( + self._assistant_model, torch_dtype=torch_dtype, **model_kwargs + ) + assistant_model.to(device) + self._generate_kwargs["assistant_model"] = assistant_model + + def transcribe( + self, + audio_files: List[Path], + batch_processor: BatchProcessor = None, + batches_queue: Queue = None, + verbose: bool = False, + ) -> Union[List[List[dict]], None]: + """ + Transcribe the given audio files. The transcriptions will be sent to a queue or a batch processor for further + processing like writing to text files. If no queue or batch processor is given, the transcriptions outputs from + the pipeline will be returned. Otherwise, `None` is returned. + + :param audio_files: The audio files to transcribe. + :param batch_processor: A batch processor. + :param batches_queue: A multiprocessing queue to put the batches in. + :param verbose: Whether to show a progress bar. Default is False. + + :returns: The transcriptions outputs from the pipeline if no queue or batch processor is given, otherwise, + `None`. + """ + # Wrap the audio files with a function to iterate over them via a generator (save memory and runtime with + # Huggingface's pipelines as they preload each input while inference is running): + def audio_iterator() -> Generator[Union[dict, str], None, None]: + if self._per_channel_transcription: + for audio_file in audio_files: + audio, sampling_rate = torchaudio.load(str(audio_file)) + audio = audio.numpy() + for channel in audio: + yield {"raw": channel, "sampling_rate": sampling_rate} + else: + for audio_file in audio_files: + yield str(audio_file) + + # Create a batch iterator: + def batch_iterator() -> Generator[List[Union[dict, str]], None, None]: + batch = [] + for audio in audio_iterator(): + batch.append(audio) + if len(batch) == self._batch_size: + yield batch + batch = [] + if batch: + yield batch + + # Prepare the successes dataframe and errors dictionary to be returned: + outputs = [] + + # Infer through the pipeline: + for input_batch in tqdm( + batch_iterator() if self._batch_size > 1 else audio_iterator(), + desc="Transcribing", + unit="channel" if self._per_channel_transcription else "audio file", + total=( + ( + (len(audio_files) // self._batch_size) + + (len(audio_files) % self._batch_size != 0) + ) + * (self._per_channel_transcription or 1) + ), + disable=not verbose, + ): + # Infer: + try: + output_batch = self._transcription_pipeline( + input_batch, + generate_kwargs=self._generate_kwargs, + ) + except Exception as exception: + # Collect the exception: + output_batch = str(exception) + # Align to batch size: + output_batch = ( + [output_batch] * len(input_batch) + if isinstance(input_batch, list) + else [output_batch] + ) + # To align with batching, if batch size is 1, wrap the output with a list: + if isinstance(output_batch, dict): + output_batch = [output_batch] + # If a batch processor is given, process the batch: + if batch_processor: + # Process it directly: + batch_processor.process_batch(batch=output_batch) + batch_processor.do_tasks() + elif batches_queue: + # Otherwise, queue the batch: + batches_queue.put(output_batch) + else: + # Otherwise, collect the output as is without processing: + outputs.append(output_batch) + + # Check if given a multiprocessing queue or a batch processor: + if batches_queue: + batches_queue.put(_MULTIPROCESSING_STOP_MARK) + + return outputs if not batch_processor else None + + +#: The value to send into multiprocessing queues to stop the process: +_MULTIPROCESSING_STOP_MARK = "STOP" + + +def _multiprocessing_process_batches( + batch_processor: BatchProcessor, + batches_queue: Queue, + tasks_queue: Queue, + n_task_completers: int, +): + """ + Process the batches in the given batches queue and put the tasks in the given tasks queue. The function will stop + when the given batches queue will receive the stop mark. It is aimed to be used with multiprocessing as a process. + + :param batch_processor: A batch processor to process the batches. + :param batches_queue: A queue to get the batches from. + :param tasks_queue: A queue to put the tasks in. + :param n_task_completers: The number of task completers (processes that run the `_multiprocessing_complete_tasks` + function). A stop mark will be sent to the tasks queue for each task completer. + """ + while True: + # Get the batch: + batch: List[dict] = batches_queue.get() + if batch == _MULTIPROCESSING_STOP_MARK: + break + + # Process the batch: + batch_processor.process_batch(batch=batch) + + # Get the tasks: + tasks = batch_processor.get_tasks() + + # Queue the tasks: + for task in tasks: + tasks_queue.put(task.to_tuple()) + + # Mark the end of the batches: + for _ in range(n_task_completers): + tasks_queue.put(_MULTIPROCESSING_STOP_MARK) + + +def _multiprocessing_complete_tasks(tasks_queue: Queue, results_queue: Queue): + """ + Complete the tasks in the given queue and put the results in the given results queue. The function will stop when + the given tasks queue will receive the stop mark. It is aimed to be used with multiprocessing as a process. + + :param tasks_queue: A queue to get the tasks from. + :param results_queue: A queue to put the results in. + """ + tasks_map = { + BaseTask.__name__: BaseTask, + SpeechDiarizationTask.__name__: SpeechDiarizationTask, + SpeechDiarizationPerChannelTask.__name__: SpeechDiarizationPerChannelTask, + } + + while True: + # Get the task: + task = tasks_queue.get() + if task == _MULTIPROCESSING_STOP_MARK: + break + + # Reconstruct the task: + task_class, task_kwargs = task + task = tasks_map[task_class](**task_kwargs) + + # Complete the task: + task.do_task() + results_queue.put((task.is_failed(), task.get_result())) + + # Mark the end of the tasks: + results_queue.put(_MULTIPROCESSING_STOP_MARK) + # Get the global logger: _LOGGER = logging.getLogger() @@ -55,7 +983,7 @@ def wrapper(**kwargs): continue if isinstance(input_argument, str): input_argument = _get_audio_files( - data_path=pathlib.Path(input_argument).absolute() + data_path=Path(input_argument).absolute() ) if len(input_argument) < size: raise ValueError( @@ -86,17 +1014,51 @@ def wrapper(**kwargs): # Run the worker: output = handler(**kwargs) + # Save the output directory of this worker: + output_directory = Path(output[0]) + # Send the output to the root rank (rank #0): output = comm.gather(output, root=0) + + # Join the data from all workers: if rank == 0: - # Join the outputs: context.logger.info("Collecting data from workers to root worker.") - output_directory = output[0][0] + + # Check if there are different output directories: + output_directories = set([Path(out_dir) for out_dir, _, _ in output]) + for r in range(1, size): + # True means the other workers should pass their files to the root worker (rank 0): + comm.send(len(output_directories) != 1, dest=r) + + # If there are different output directories, listen to the other workers: + if len(output_directories) != 1: + # Collect the files from the other workers: + files = [] + for r in range(1, size): + files.extend(comm.recv(source=r)) + # Write the files to the root worker's output directory: + for file_name, file_content in files: + with open(output_directory / file_name, "w") as f: + f.write(file_content) + + # Concatenate the dataframes: dataframe = pd.concat(objs=[df for _, df, _ in output], axis=0) + + # Concatenate the errors dictionaries: errors_dictionary = reduce( operator.ior, [err for _, _, err in output], {} ) - return output_directory, dataframe, errors_dictionary + + return str(output_directory), dataframe, errors_dictionary + + # Listen to rank 0 to see if there are different output directories and this rank need to send its files to + # it: + if comm.recv(source=0): + files = [] + for file in os.listdir(output_directory): + with open(output_directory / file, "r") as f: + files.append((file, f.read())) + comm.send(files, dest=0) return None return wrapper @@ -133,165 +1095,245 @@ def _check_mlrun_and_open_mpi() -> Tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intrac @open_mpi_handler(worker_inputs=["data_path"], root_worker_inputs={"verbose": True}) def transcribe( - data_path: Union[str, List[str]], - output_directory: str, - model_name: str = "base", - device: Literal["cuda", "cpu", "auto"] = "auto", - compute_type: str = "default", - language: str = None, + # Input / Output kwargs: + data_path: Union[str, Path, List[Union[str, Path]]], + output_directory: str = None, + # Model loading kwargs: + model_name: str = "openai/whisper-tiny", + device: str = None, + use_flash_attention_2: bool = None, + use_better_transformers: bool = None, + # Generation kwargs: + assistant_model: str = None, + max_new_tokens: int = 128, + chunk_length_s: int = 30, + batch_size: int = 8, + spoken_language: str = None, translate_to_english: bool = False, + # Diarization kwargs: speech_diarization: Dict[str, List[Tuple[float, float, str]]] = None, - audio_duration: bool = False, - init_kwargs: dict = None, - transcribe_kwargs: dict = None, + speech_diarize_per_channel: int = None, + speaker_labels: List[str] = None, + # Other kwargs: + use_multiprocessing: Union[bool, int] = False, verbose: bool = False, -) -> Tuple[str, pd.DataFrame, dict]: +): """ Transcribe audio files into text files and collect additional data. The end result is a directory of transcribed text files and a dataframe containing the following columns: * audio_file - The audio file path. * transcription_file - The transcribed text file name in the output directory. - * language - The detected language in the audio file. - * language_probability - The detected language probability. - * duration - The duration (in seconds) of the audio file (only if `audio_duration` is set to True). - - :param data_path: A directory of audio files or a single file or a list of files to transcribe. - :param output_directory: Path to a directory to save all transcribed audio files. - :param model_name: One of the official model names of Whisper: {'tiny.en', 'tiny', 'base.en', 'base', - 'small.en', 'small', 'medium.en', 'medium', 'large-v1', 'large-v2', 'large'} or a - full name of a fine-tuned whisper model from the huggingface hub. - :param device: Device to load the model. Can be one of {"cuda", "cpu"}. Default will prefer "cuda" - if available. To use a specific GPU or more than one GPU, pass the `device_index` - argument via the `init_kwargs`. - :param compute_type: The data type to use for computation. For more information, check - https://opennmt.net/CTranslate2/quantization.html. Default: "default" - will use the - default type depending on the device used. - :param language: The spoken language to force Whisper the output language. If None, the Whisper model - will automatically predict the output langauge. Default: None. - :param translate_to_english: Whether to translate the English post transcription. Default: False. - :param speech_diarization: A speech diarization dictionary with the file names to transcribe as keys and their - diarization as value. The diarization is a list of tuples: (start, end, speaker). - The transcription result will be in the following format: - "{speaker}: text text text.". Files with missing diarizations will print a warning. - Pay attention the diarization must be for the entire duration of the audio file (as - long as Whisper is predicting words up until then). - :param audio_duration: Whether to include the audio files duration (in seconds). The estimated duration is - from bitrate and may be inaccurate. Default: False. - :param init_kwargs: Additional `WhisperModel.__init__` keyword arguments to use. - :param transcribe_kwargs: Additional `WhisperModel.transcribe` keyword arguments to use. - :param verbose: Whether to present logs of a progress bar and errors. Default: False. - - :returns: A tuple of: - - * Path to the output directory. - * A dataframe dataset of the transcribed file names. - * A dictionary of errored files that were not transcribed. + + The transcription is based on Huggingface's ASR pipeline - + https://huggingface.co/transformers/main_classes/pipelines.html#transformers.AutomaticSpeechRecognitionPipeline and + is tested with OpenAI's Whisper models - https://huggingface.co/openai. + + If one of the speaker diarization parameters are given (either `speech_diarization` or + `speech_diarize_per_channel`), the transcription will be written in a conversation format, where each speaker will + be written in a separate line:: + + speaker_1: text + speaker_2: text + speaker_1: text + ... + + :param data_path: A directory of audio files or a single file or a list of files to transcribe. + :param output_directory: Path to a directory to save all transcribed audio files. If not given, will save + the transcribed files in a temporary directory. + :param model_name: The model name to use. Should be a model from the OpenAI's Whisper models for + best results (for example "tiny", "base", "large", etc.). See here for more + information: https://huggingface.co/openai?search_models=whisper. + :param device: The device to use for inference. If not given, will use GPU if available. + :param use_flash_attention_2: Whether to use the Flash Attention 2 implementation. It can be used only with + one of the following GPUs: Nvidia H series and Nvidia A series. T4 support + will be available soon. + + Note: If both `use_flash_attention_2` and + `use_better_transformers` are `None`, the optimization will be chosen + automatically according to the available resources. + + :param use_better_transformers: Whether to use the Better Transformers library to further optimize the model. + Should be used for all use cases that do not support flash attention 2. + + Note: If both `use_flash_attention_2` and `use_better_transformers` are + `None`, the optimization will be chosen automatically according to the + available resources. + :param assistant_model: The assistant model name to use for inference. Notice that the optimizations + (flash attention 2 and better transformers) will be applied for the assistant as + well. Should be a model from Huggingface's distil-whisper (see here for more + information: https://github.com/huggingface/distil-whisper). + + Note: Currently an assistant model is only usable with batch size of 1. + :param max_new_tokens: The maximum number of new tokens to generate. This is used to limit the + generation length. Default is 128 tokens. + :param chunk_length_s: The audio chunk to split the audio to (in seconds). Default is 30 seconds. + :param batch_size: The batch size to use for inference. Default is 2. + :param spoken_language: Aim whisper to know what language is spoken. If None, it will try to detect + it. + :param translate_to_english: Whether to translate the transcriptions to English. + :param speech_diarization: A speech diarization dictionary with the file names to transcribe as keys and + their diarization as value. The diarization is a list of tuples: + (start, end, speaker). An example + for a diarization dictionary:: + + { + "audio_file_name": [ + { + "start": 0.0, + "end": 2.0, + "speaker": "Agent", + }, + { + "start": 2.0, + "end": 4.0, + "speaker": "Client", + }, + ... + ], + ... + } + + Note: The diarization must be for the entire duration of the audio file (as long + as Whisper is predicting words up until then. + :param speech_diarize_per_channel: Perform speech diarization per channel. Each speaker is expected to belong to + a separate channel in the audio. Notice: This will make the transcription + slower as each channel wil be transcribed separatly. If a speech diarization + is passed (via the `speech_diarization` parameter), this parameter is + ignored. + :param speaker_labels: A list of speaker labels by channel order to use for writing the + transcription with respect to per channel speech diarization. This won't be + used together with a given speech diarization (via the `speech_diarization` + parameter). + :param use_multiprocessing: Whether to use multiprocessing to transcribe the audio files. Can be either a + boolean value or an integer. If `True`, will use the default amount of workers + (3): 1 for transcription, 1 for batch processing and 1 for task completion (such + as speech diarization and writing to files). To control the amount of tasks + completion workers, an integer can be provided to specify the amount of workers. + `False`, will use a single process. Default is `False`. + :param verbose: Whether to print the progress of the transcription. Default is `False`. """ global _LOGGER # Get the input audio files to transcribe: if verbose: _LOGGER.info("Collecting audio files.") - if isinstance(data_path, str): - data_path = pathlib.Path(data_path).absolute() - audio_files = _get_audio_files(data_path=data_path) - else: - audio_files = data_path + audio_files = _get_audio_files(data_path=data_path) if verbose: _LOGGER.info(f"Collected {len(audio_files)} audio files.") - # Load the whisper model: + # Get the output directory: + if output_directory is None: + if verbose: + _LOGGER.info("No output directory given, using temporary directory.") + output_directory = tempfile.mkdtemp() + output_directory = Path(output_directory).absolute() + output_directory.mkdir(exist_ok=True, parents=True) if verbose: - _LOGGER.info(f"Loading model '{model_name}' - using device '{device}'.") - init_kwargs = init_kwargs or {} - model = faster_whisper.WhisperModel( - model_size_or_path=model_name, + _LOGGER.info(f"Transcriptions will be saved to: {output_directory}") + + # Initialize a batch processor according to user requirements (no speech diarization, given speech diarization, + # speech diarization per channel): + if speech_diarization: + batch_processor = SpeechDiarizationBatchProcessor( + audio_files=audio_files, + output_directory=output_directory, + speech_diarization=speech_diarization, + ) + elif speech_diarize_per_channel: + batch_processor = PerChannelSpeechDiarizationBatchProcessor( + audio_files=audio_files, + output_directory=output_directory, + n_channels=speech_diarize_per_channel, + speakers=speaker_labels, + ) + else: + batch_processor = BatchProcessor( + audio_files=audio_files, + output_directory=output_directory, + ) + + # Initialize the transcription pipeline: + transcriber = Transcriber( device=device, - compute_type=compute_type, - **init_kwargs, + use_flash_attention_2=use_flash_attention_2, + use_better_transformers=use_better_transformers, + assistant_model=assistant_model, + model_name=model_name, + max_new_tokens=max_new_tokens, + chunk_length_s=chunk_length_s, + batch_size=batch_size, + return_timestamps=( + "word" + if speech_diarization is not None or speech_diarize_per_channel is not None + else False + ), + per_channel_transcription=speech_diarize_per_channel or 0, + spoken_language=spoken_language, + translate_to_english=translate_to_english, ) - if verbose: - _LOGGER.info(f"Model loaded successfully.") - # Prepare the successes dataframe and errors dictionary to be returned: + # Run the transcription: + if use_multiprocessing: + results = _parallel_run( + n_workers=use_multiprocessing + if isinstance(use_multiprocessing, int) + else 1, + audio_files=audio_files, + batch_processor=batch_processor, + transcriber=transcriber, + verbose=verbose, + ) + else: + results = _run( + audio_files=audio_files, + batch_processor=batch_processor, + transcriber=transcriber, + verbose=verbose, + ) + + # Process the results: + if verbose: + _LOGGER.info("Summarizing the results.") successes = [] errors = {} - - # Create the output directory: - output_directory = pathlib.Path(output_directory) - output_directory.mkdir(parents=True, exist_ok=True) - - # Prepare the transcribe keyword arguments: - transcribe_kwargs = transcribe_kwargs or {} - transcribe_kwargs["language"] = language - transcribe_kwargs["task"] = "translate" if translate_to_english else "transcribe" - - # Go over the audio files and transcribe: - for audio_file in tqdm( - audio_files, desc="Transcribing", unit="file", disable=not verbose - ): - try: - # Transcribe: - transcription_and_info = _transcribe( - audio_file=audio_file, - model=model, - transcribe_kwargs=transcribe_kwargs, - speech_diarization=_get_diarization( # Get the diarization (if provided). - speech_diarization=speech_diarization, - file_name=audio_file.name, - verbose=verbose, - ), - audio_duration=audio_duration, - ) - # Write the transcription to file: - transcription_file = _save_to_file( - transcription=transcription_and_info[0], - file_name=audio_file.stem, - output_directory=output_directory, - ) - # Note as a success in the list: - successes.append( - [ - audio_file.name, - transcription_file.name, - *transcription_and_info[1:], - ] - ) - except Exception as exception: - # Note the exception as error in the dictionary: - if verbose: - _LOGGER.warning(f"Error in file: '{audio_file.name}'") - errors[str(audio_file.name)] = str(exception) - continue - - # Construct the transcriptions dataframe: - columns = [ - "audio_file", - "transcription_file", - "language", - "language_probability", - ] - if audio_duration: - columns.append("duration") - successes = pd.DataFrame( - successes, - columns=columns, - ) - - # Print the head of the produced dataframe and return: + for is_error, result in results: + if is_error: + errors[result[0]] = result[1] + else: + successes.append(result) + successes = pd.DataFrame(successes, columns=["audio_file", "transcription_file"]) if verbose: _LOGGER.info( f"Done ({successes.shape[0]}/{len(audio_files)})\n" f"Transcriptions summary:\n" f"{successes.head()}" ) + return str(output_directory), successes, errors def _get_audio_files( - data_path: pathlib.Path, -) -> List[pathlib.Path]: + data_path: Union[Path, str, list], +) -> List[Path]: + """ + Get the audio files to transcribe. If a path to a directory is given, all files in the directory will be collected. + + :param data_path: The data path to collect the audio files from. + + :returns: The audio files list. + """ + # Check if given a list of paths: + if isinstance(data_path, list): + audio_files = [] + for path in data_path: + audio_files.extend(_get_audio_files(data_path=path)) + return audio_files + + # Check if given a single string path to cast it to a `pathlib.Path`: + if isinstance(data_path, str): + data_path = Path(data_path).absolute() + # Check if the path is of a directory or a file: if data_path.is_dir(): # Get all files inside the directory: @@ -300,190 +1342,123 @@ def _get_audio_files( audio_files = [data_path] else: raise ValueError( - f"Unrecognized data path. The parameter `data_path` must be either a directory path or a file path. " - f"Given: {str(data_path)} " + f"Unrecognized data path. The parameter `data_path` must be a valid path to either a directory path or a " + f"file. Given: {str(data_path)} " ) return audio_files -class _DiarizationSegment(NamedTuple): - start: float - end: float - speaker: str +def _run( + audio_files: List[Path], + batch_processor: BatchProcessor, + transcriber: Transcriber, + verbose: bool, +) -> List[Tuple[bool, Tuple[str, str]]]: + """ + Run the transcription without multiprocessing. + :param audio_files: The audio files to transcribe. + :param batch_processor: The batch processor to use. + :param transcriber: The transcriber to use. + :param verbose: Verbosity. -def _get_diarization( - speech_diarization: Dict[str, List[Tuple[float, float, str]]], - file_name: str, - verbose: bool, -) -> Union[List[_DiarizationSegment], None]: - diarization = None - if speech_diarization is not None: - diarization = speech_diarization.get(file_name) - if diarization is None: - if verbose: - _LOGGER.warning( - f"Missing speech diarization for the audio file '{file_name}'. Continuing transcribing without " - f"diarization." - ) - diarization = [_DiarizationSegment(*segment) for segment in diarization] - return diarization - - -def _get_next_diarization_segment( - word: faster_whisper.transcribe.Word, - speech_diarization: List[_DiarizationSegment], - last_chosen_index: int, -) -> int: - # Get the last chosen diarization segment: - last_chosen = speech_diarization[last_chosen_index] - - # If the last chosen segment is the last segment, return it: - if last_chosen_index == len(speech_diarization) - 1: - return last_chosen_index - - # If the word ends before the last chosen segment: - if word.end <= last_chosen.start: - # Then it is still the closest segment - return last_chosen_index - - # We check if it ends inside the last chosen segment: - if word.end < last_chosen.end: - # Then it still is the closest segment - return last_chosen_index - - # The word ends after the segment, we need to collect all next segments up until the word ends before them: - possible_segments = [last_chosen_index] - for i in range(last_chosen_index + 1, len(speech_diarization)): - if word.end > speech_diarization[i].end: - possible_segments.append(i) - continue - possible_segments.append(i) - break - - # Check for the most overlapping option: - best_overlap = 0 - overlapping_segment = None - for i in possible_segments: - overlap = 0 - # If the word starts before segment: - if word.start <= speech_diarization[i].start: - # If it ends before the segment, there is an overlap from the start of the segment to the end of the word: - if word.end < speech_diarization[i].end: - overlap = word.end - speech_diarization[i].start - else: - # The word is wrapping the segment, the overlap is the segment's length: - overlap = speech_diarization[i].end - speech_diarization[i].start - # The word starts in segment, check if the word ends in it: - elif word.end < speech_diarization[i].end: - # The overlap is the word's length: - overlap = word.end - word.start - # The word start in segment but ends after it, the overlap is from the word's start to the segment's end: - else: - overlap = speech_diarization[i].end - word.start - # Check for new best overlap: - if overlap > best_overlap: - best_overlap = overlap - overlapping_segment = i - if overlapping_segment is not None: - return overlapping_segment - - # If there is no overlapping segment, return the closest segment: - best_distance = None - closest_segment = None - for i in possible_segments: - distance = ( - word.start - speech_diarization[i].end - if word.start > speech_diarization[i].end - else speech_diarization[i].start - word.end - ) - if best_distance is None or distance < best_distance: - best_distance = distance - closest_segment = i - return closest_segment - - -def _construct_transcription( - segments: List[faster_whisper.transcribe.Segment], - speech_diarization: List[_DiarizationSegment], -) -> str: - # If there is no diarization, concatenate all segments and return: - if speech_diarization is None: - return " ".join([segment.text for segment in segments]) - - # There is a diarization, try to match the Whisper model predicted timestamps to the closest diarization segment - # (closest diarization segment will be the most overlapping with the word, and if there is no overlap, the closest - # segment to the word): - diarization_index = 0 - speaker = speech_diarization[diarization_index].speaker - text = f"{speaker}:" - for segment in segments: - for word in segment.words: - # Get the next diarization segment: - diarization_index = _get_next_diarization_segment( - word=word, - speech_diarization=speech_diarization, - last_chosen_index=diarization_index, - ) - # Check if the segment is of the same speaker: - if speech_diarization[diarization_index].speaker == speaker: - # Collect the word: - text += word.word - else: - # Append a newline and update the new speaker: - speaker = speech_diarization[diarization_index].speaker - text += f"\n{speaker}:{word.word}" - - return text - - -def _transcribe( - audio_file: pathlib.Path, - model: faster_whisper.WhisperModel, - transcribe_kwargs: dict, - speech_diarization: List[_DiarizationSegment], - audio_duration: bool, -) -> Union[Tuple[str, str, float], Tuple[str, str, float, float]]: - # Transcribe (Segments is a generator, so we cast to list to begin transcription from start to end): - segments, info = model.transcribe( - audio=str(audio_file), - **transcribe_kwargs, - word_timestamps=speech_diarization is not None, + :returns: The collected results. + """ + # Load the transcription pipeline: + if verbose: + _LOGGER.info(f"Loading the transcription pipeline.") + transcriber.load() + if verbose: + _LOGGER.info("Transcription pipeline loaded.") + + # Transcribe the files: + transcriber.transcribe( + audio_files=audio_files, + batch_processor=batch_processor, + verbose=verbose, ) - segments = list(segments) - # Check if speech diarization was provided: - if speech_diarization is None: - text = "".join([segment.text for segment in segments]) - else: - text = _construct_transcription( - segments=segments, - speech_diarization=speech_diarization, + # Return the results: + return batch_processor.get_results() + + +def _parallel_run( + n_workers: int, + audio_files: List[Path], + batch_processor: BatchProcessor, + transcriber: Transcriber, + verbose: bool, +): + """ + Run the transcription with multiprocessing. + + :param n_workers: The amount of workers to use as task completers. + :param audio_files: The audio files to transcribe. + :param batch_processor: The batch processor to use. + :param transcriber: The transcriber to use. + :param verbose: Verbosity. + + :returns: The collected results. + """ + # Initialize the multiprocessing queues: + batches_queue = Queue() + tasks_queue = Queue() + results_queue = Queue() + + # Initialize the multiprocessing processes: + batch_processing_process = Process( + target=_multiprocessing_process_batches, + kwargs={ + "batch_processor": batch_processor, + "batches_queue": batches_queue, + "tasks_queue": tasks_queue, + "n_task_completers": n_workers, + }, + ) + task_completion_processes = [ + Process( + target=_multiprocessing_complete_tasks, + kwargs={"tasks_queue": tasks_queue, "results_queue": results_queue}, ) - text = text.strip() + for _ in range(n_workers) + ] - # Return the transcription text and the additional information: - if audio_duration: - return text.strip(), info.language, info.language_probability, info.duration - return text.strip(), info.language, info.language_probability + # Start the multiprocessing processes: + batch_processing_process.start() + for p in task_completion_processes: + p.start() + # Load the transcription pipeline: + if verbose: + _LOGGER.info(f"Loading the transcription pipeline.") + transcriber.load() + if verbose: + _LOGGER.info("Transcription pipeline loaded.") -def _save_to_file( - transcription: str, file_name: str, output_directory: pathlib.Path -) -> pathlib.Path: - # Prepare the file full path (checking for no duplications): - transcription_file = output_directory / f"{file_name}.txt" - i = 1 - while transcription_file.exists(): - i += 1 - transcription_file = output_directory / f"{file_name}_{i}.txt" + # Transcribe the files: + transcriber.transcribe( + audio_files=audio_files, batches_queue=batches_queue, verbose=verbose + ) - # Make sure all directories are created: - transcription_file.parent.mkdir(exist_ok=True, parents=True) + # Collect the results: + results = [] + stop_marks_counter = 0 + while True: + # Get a result from the queue: + result: Tuple[bool, Tuple[str, str]] = results_queue.get() + if result == _MULTIPROCESSING_STOP_MARK: + stop_marks_counter += 1 + if stop_marks_counter == n_workers: + break + else: + # Collect the result: + results.append(result) - # Write to file: - with open(transcription_file, "w") as fp: - fp.write(transcription) + # Wait for the processes to finish: + results_queue.empty() + batch_processing_process.join() + for p in task_completion_processes: + p.join() - return transcription_file + return results \ No newline at end of file