diff --git a/auto_trainer/auto_trainer.py b/auto_trainer/auto_trainer.py index cb72cc2db..abaa63578 100755 --- a/auto_trainer/auto_trainer.py +++ b/auto_trainer/auto_trainer.py @@ -72,10 +72,9 @@ def _get_dataframe( # Getting the dataset: if mlrun.utils.StorePrefix.FeatureVector == store_uri_prefix: # FeatureVector case: + fv = mlrun.datastore.get_store_resource(dataset.artifact_url) label_columns = label_columns or dataset.meta.status.label_column - dataset = fs.get_offline_features( - dataset.meta.uri, drop_columns=drop_columns - ).to_dataframe() + dataset = fv.get_offline_features(drop_columns=drop_columns).to_dataframe() context.logger.info(f"label columns: {label_columns}") diff --git a/auto_trainer/function.yaml b/auto_trainer/function.yaml index f4d090d31..9190ac7a8 100644 --- a/auto_trainer/function.yaml +++ b/auto_trainer/function.yaml @@ -2,7 +2,7 @@ kind: job metadata: name: auto-trainer tag: '' - hash: 95201dacf05c181e0257b5dd145cb79a503b6fbe + hash: 40840bc9015f96dfed199b777942398954578c5d project: '' labels: author: yonish @@ -14,7 +14,7 @@ spec: args: [] image: mlrun/mlrun build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICBzdG9yZV91cmlfcHJlZml4LCBfID0gbWxydW4uZGF0YXN0b3JlLnBhcnNlX3N0b3JlX3VyaShkYXRhc2V0LmFydGlmYWN0X3VybCkKCiAgICAjIEdldHRpbmcgdGhlIGRhdGFzZXQ6CiAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgIyBGZWF0dXJlVmVjdG9yIGNhc2U6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IGxhYmVsX2NvbHVtbnMgb3IgZGF0YXNldC5tZXRhLnN0YXR1cy5sYWJlbF9jb2x1bW4KICAgICAgICBkYXRhc2V0ID0gZnMuZ2V0X29mZmxpbmVfZmVhdHVyZXMoCiAgICAgICAgICAgIGRhdGFzZXQubWV0YS51cmksIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMKICAgICAgICApLnRvX2RhdGFmcmFtZSgpCgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQoKICAgIGVsaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMgbm90IHByb3ZpZGVkLCBtYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBlbGlmIGlzaW5zdGFuY2UoZGF0YXNldCwgKGxpc3QsIGRpY3QpKToKICAgICAgICAjIGxpc3QvZGljdCBjYXNlOgogICAgICAgIGRhdGFzZXQgPSBwZC5EYXRhRnJhbWUoZGF0YXNldCkKICAgICAgICAjIENoZWNraW5nIGlmIGRyb3BfY29sdW1ucyBwcm92aWRlZCBieSBpbnRlZ2VyIHR5cGU6CiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgc3RyKSBvciAoCiAgICAgICAgICAgICAgICBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgbGlzdCkKICAgICAgICAgICAgICAgIGFuZCBhbnkoaXNpbnN0YW5jZShjb2wsIHN0cikgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpCiAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIG11c3QgYmUgYW4gaW50ZWdlci9saXN0IG9mIGludGVnZXJzIGlmIG5vdCBwcm92aWRlZCB3aXRoIGEgVVJJL0ZlYXR1cmVWZWN0b3IgZGF0YXNldCIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xLCBpbnBsYWNlPVRydWUpCgogICAgZWxzZToKICAgICAgICAjIHNpbXBsZSBVUkwgY2FzZToKICAgICAgICBkYXRhc2V0ID0gZGF0YXNldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgIm5vdCBhbGwgb2YgdGhlIGNvbHVtbnMgdG8gZHJvcCBpbiB0aGUgZGF0YXNldCwgZHJvcCBjb2x1bW5zIHByb2Nlc3Mgc2tpcHBlZCIKICAgICAgICAgICAgICAgICkKCiAgICByZXR1cm4gZGF0YXNldCwgbGFiZWxfY29sdW1ucwoKCmRlZiB0cmFpbigKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBtb2RlbF9jbGFzczogc3RyLAogICAgbGFiZWxfY29sdW1uczogT3B0aW9uYWxbVW5pb25bc3RyLCBMaXN0W3N0cl1dXSA9IE5vbmUsCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSAibW9kZWwiLAogICAgdGFnOiBzdHIgPSAiIiwKICAgIHNhbXBsZV9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRlc3Rfc2V0OiBEYXRhSXRlbSA9IE5vbmUsCiAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemU6IGZsb2F0ID0gTm9uZSwKICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gTm9uZSwKICAgIGxhYmVsczogZGljdCA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgVHJhaW5pbmcgYSBtb2RlbCB3aXRoIHRoZSBnaXZlbiBkYXRhc2V0LgoKICAgIGV4YW1wbGU6OgoKICAgICAgICBpbXBvcnQgbWxydW4KICAgICAgICBwcm9qZWN0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9wcm9qZWN0KCJteS1wcm9qZWN0IikKICAgICAgICBwcm9qZWN0LnNldF9mdW5jdGlvbigiaHViOi8vYXV0b190cmFpbmVyIiwgInRyYWluIikKICAgICAgICB0cmFpbmVyX3J1biA9IHByb2plY3QucnVuKAogICAgICAgICAgICBuYW1lPSJ0cmFpbiIsCiAgICAgICAgICAgIGhhbmRsZXI9InRyYWluIiwKICAgICAgICAgICAgaW5wdXRzPXsiZGF0YXNldCI6ICIuL3BhdGgvdG8vZGF0YXNldC5jc3YifSwKICAgICAgICAgICAgcGFyYW1zPXsKICAgICAgICAgICAgICAgICJtb2RlbF9jbGFzcyI6ICJza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb24iLAogICAgICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMiOiAibGFiZWwiLAogICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyI6ICJpZCIsCiAgICAgICAgICAgICAgICAibW9kZWxfbmFtZSI6ICJteS1tb2RlbCIsCiAgICAgICAgICAgICAgICAidGFnIjogInYxLjAuMCIsCiAgICAgICAgICAgICAgICAic2FtcGxlX3NldCI6ICIuL3BhdGgvdG8vc2FtcGxlX3NldC5jc3YiLAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IjogIi4vcGF0aC90by90ZXN0X3NldC5jc3YiLAogICAgICAgICAgICAgICAgIkNMQVNTX3NvbHZlciI6ICJsaWJsaW5lYXIiLAogICAgICAgICAgICB9LAogICAgICAgICkKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQKICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIG1vZGVsX2NsYXNzOiAgICAgICAgICAgICBUaGUgY2xhc3Mgb2YgdGhlIG1vZGVsLCBlLmcuIGBza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb25gCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uczogICAgICAgICAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuIE1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyIG9yIGEgbGlzdCBvZiBzdHJpbmdzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW5zIHRvIGRyb3AKICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgbmFtZSB0byB1c2UgZm9yIHN0b3JpbmcgdGhlIG1vZGVsIGFydGlmYWN0LCBkZWZhdWx0IHRvICdtb2RlbCcKICAgIDpwYXJhbSB0YWc6ICAgICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgdGFnIHRvIGxvZyB3aXRoCiAgICA6cGFyYW0gc2FtcGxlX3NldDogICAgICAgICAgICAgIEEgc2FtcGxlIHNldCBvZiBpbnB1dHMgZm9yIHRoZSBtb2RlbCBmb3IgbG9nZ2luZyBpdHMgc3RhdHMgYWxvbmcgdGhlIG1vZGVsIGluIGZhdm91cgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvZiBtb2RlbCBtb25pdG9yaW5nLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIHRlc3Rfc2V0OiAgICAgICAgICAgICAgICBUaGUgdGVzdCBzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIHdpdGguCiAgICA6cGFyYW0gdHJhaW5fdGVzdF9zcGxpdF9zaXplOiAgIGlmIHRlc3Rfc2V0IHdhcyBwcm92aWRlZCB0aGVuIHRoaXMgYXJndW1lbnQgaXMgaWdub3JlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgU2hvdWxkIGJlIGJldHdlZW4gMC4wIGFuZCAxLjAgYW5kIHJlcHJlc2VudCB0aGUgcHJvcG9ydGlvbiBvZiB0aGUgZGF0YXNldCB0byBpbmNsdWRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGluIHRoZSB0ZXN0IHNwbGl0LiBUaGUgc2l6ZSBvZiB0aGUgVHJhaW5pbmcgc2V0IGlzIHNldCB0byB0aGUgY29tcGxlbWVudCBvZiB0aGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHZhbHVlLiBEZWZhdWx0ID0gMC4yCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICAgICAgIFJlbGV2YW50IG9ubHkgd2hlbiB1c2luZyB0cmFpbl90ZXN0X3NwbGl0X3NpemUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgcmFuZG9tIHN0YXRlIHNlZWQgdG8gc2h1ZmZsZSB0aGUgZGF0YS4gRm9yIG1vcmUgaW5mb3JtYXRpb24sIHNlZToKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tX3N0YXRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE5vdGljZSB0aGF0IGhlcmUgd2Ugb25seSBwYXNzIGludGVnZXIgdmFsdWVzLgogICAgOnBhcmFtIGxhYmVsczogICAgICAgICAgICAgICAgICBMYWJlbHMgdG8gbG9nIHdpdGggdGhlIG1vZGVsCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHdpdGggcHJlZml4ZXMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoYXQgd2lsbCBiZSBwYXJzZWQgYW5kIHBhc3NlZCB0byB0aGUgcmVsZXZhbnQgZnVuY3Rpb24sIGJ5IHRoZSBmb2xsb3dpbmcgcHJlZml4ZXM6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYENMQVNTX2AgLSBmb3IgdGhlIG1vZGVsIGNsYXNzIGFyZ3VtZW50cwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIGBGSVRfYCAtIGZvciB0aGUgYGZpdGAgZnVuY3Rpb24gYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYFRSQUlOX2AgLSBmb3IgdGhlIGB0cmFpbmAgZnVuY3Rpb24gKGluIHhnYiBvciBsZ2JtIHRyYWluIGZ1bmN0aW9uIC0gZnV0dXJlKQoKICAgICIiIgogICAgIyBWYWxpZGF0ZSBpbnB1dHM6CiAgICAjIENoZWNrIGlmIGV4YWN0bHkgb25lIG9mIHRoZW0gaXMgc3VwcGxpZWQ6CiAgICBpZiB0ZXN0X3NldCBpcyBOb25lOgogICAgICAgIGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBpcyBOb25lOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IG9yIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBhcmUgbm90IHByb3ZpZGVkLCBzZXR0aW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB0byAwLjIiCiAgICAgICAgICAgICkKICAgICAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gMC4yCgogICAgZWxpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgInRlc3Rfc2V0IHByb3ZpZGVkLCBpZ25vcmluZyBnaXZlbiB0cmFpbl90ZXN0X3NwbGl0X3NpemUgdmFsdWUiCiAgICAgICAgKQogICAgICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSA9IE5vbmUKCiAgICAjIEdldCBEYXRhRnJhbWUgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIEdldHRpbmcgdGhlIHNhbXBsZSBzZXQ6CiAgICBpZiBzYW1wbGVfc2V0IGlzIE5vbmU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgZiJTYW1wbGUgc2V0IG5vdCBnaXZlbiwgdXNpbmcgdGhlIHdob2xlIHRyYWluaW5nIHNldCBhcyB0aGUgc2FtcGxlIHNldCIKICAgICAgICApCiAgICAgICAgc2FtcGxlX3NldCA9IGRhdGFzZXQKICAgIGVsc2U6CiAgICAgICAgc2FtcGxlX3NldCwgXyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIGRhdGFzZXQ9c2FtcGxlX3NldCwKICAgICAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgICAgICkKCiAgICAjIFBhcnNpbmcga3dhcmdzOgogICAgIyBUT0RPOiBVc2UgaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24uCiAgICB0cmFpbl9rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeChzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLlRSQUlOKQogICAgZml0X2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuRklUKQogICAgbW9kZWxfY2xhc3Nfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoCiAgICAgICAgc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5NT0RFTF9DTEFTUwogICAgKQoKICAgICMgQ2hlY2sgaWYgbW9kZWwgb3IgZnVuY3Rpb246CiAgICBpZiBoYXNhdHRyKG1vZGVsX2NsYXNzLCAidHJhaW4iKToKICAgICAgICAjIFRPRE86IE5lZWQgdG8gY2FsbDogbW9kZWwoKSwgYWZ0ZXJ3YXJkcyB0byBzdGFydCB0aGUgdHJhaW4gZnVuY3Rpb24uCiAgICAgICAgIyBtb2RlbCA9IGNyZWF0ZV9mdW5jdGlvbihmInttb2RlbF9jbGFzc30udHJhaW4iKQogICAgICAgIHJhaXNlIE5vdEltcGxlbWVudGVkRXJyb3IKICAgIGVsc2U6CiAgICAgICAgIyBDcmVhdGluZyBtb2RlbCBpbnN0YW5jZToKICAgICAgICBtb2RlbCA9IGNyZWF0ZV9jbGFzcyhtb2RlbF9jbGFzcykoKiptb2RlbF9jbGFzc19rd2FyZ3MpCgogICAgeCA9IGRhdGFzZXQuZHJvcChsYWJlbF9jb2x1bW5zLCBheGlzPTEpCiAgICB5ID0gZGF0YXNldFtsYWJlbF9jb2x1bW5zXQogICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplOgogICAgICAgIHhfdHJhaW4sIHhfdGVzdCwgeV90cmFpbiwgeV90ZXN0ID0gdHJhaW5fdGVzdF9zcGxpdCgKICAgICAgICAgICAgeCwgeSwgdGVzdF9zaXplPXRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSwgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZQogICAgICAgICkKICAgIGVsc2U6CiAgICAgICAgeF90cmFpbiwgeV90cmFpbiA9IHgsIHkKCiAgICAgICAgdGVzdF9zZXQgPSB0ZXN0X3NldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICB0ZXN0X3NldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKCiAgICAgICAgeF90ZXN0LCB5X3Rlc3QgPSB0ZXN0X3NldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSksIHRlc3Rfc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgQXV0b01MUnVuLmFwcGx5X21scnVuKAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIG1vZGVsX25hbWU9bW9kZWxfbmFtZSwKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgdGFnPXRhZywKICAgICAgICBzYW1wbGVfc2V0PXNhbXBsZV9zZXQsCiAgICAgICAgeV9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgdGVzdF9zZXQ9dGVzdF9zZXQsCiAgICAgICAgeF90ZXN0PXhfdGVzdCwKICAgICAgICB5X3Rlc3Q9eV90ZXN0LAogICAgICAgIGFydGlmYWN0cz1jb250ZXh0LmFydGlmYWN0cywKICAgICAgICBsYWJlbHM9bGFiZWxzLAogICAgKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInRyYWluaW5nICd7bW9kZWxfbmFtZX0nIikKICAgIG1vZGVsLmZpdCh4X3RyYWluLCB5X3RyYWluLCAqKmZpdF9rd2FyZ3MpCgoKZGVmIGV2YWx1YXRlKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBtb2RlbDogc3RyLAogICAgZGF0YXNldDogbWxydW4uRGF0YUl0ZW0sCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBFdmFsdWF0aW5nIGEgbW9kZWwuIEFydGlmYWN0cyBnZW5lcmF0ZWQgYnkgdGhlIE1MSGFuZGxlci4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgICAgICAgICAgIFRoZSBtb2RlbCBTdG9yZSBwYXRoLgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byBldmFsdWF0ZSB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcC4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIFBhcnNpbmcgbGFiZWxfY29sdW1uczoKICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zID0gW10KICAgIGlmIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9ICgKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyBpZiBpc2luc3RhbmNlKGxhYmVsX2NvbHVtbnMsIGxpc3QpIGVsc2UgW2xhYmVsX2NvbHVtbnNdCiAgICAgICAgKQogICAgICAgIGZvciBsYyBpbiBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBpZiBmcy5jb21tb24uZmVhdHVyZV9zZXBhcmF0b3IgaW4gbGM6CiAgICAgICAgICAgICAgICBmZWF0dXJlX3NldF9uYW1lLCBsYWJlbF9uYW1lLCBhbGlhcyA9IGZzLmNvbW1vbi5wYXJzZV9mZWF0dXJlX3N0cmluZyhsYykKICAgICAgICAgICAgICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zLmFwcGVuZChhbGlhcyBvciBsYWJlbF9uYW1lKQogICAgICAgIGlmIHBhcnNlZF9sYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gcGFyc2VkX2xhYmVsX2NvbHVtbnMKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgIyBMb2FkaW5nIHRoZSBtb2RlbCBhbmQgcHJlZGljdGluZzoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbCgKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQsIG1vZGVsX25hbWU9Im1vZGVsX0xpbmVhclJlZ3Jlc3Npb24iCiAgICApCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4obW9kZWxfaGFuZGxlci5tb2RlbCwgeV90ZXN0PXksIG1vZGVsX3BhdGg9bW9kZWwpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmImV2YWx1YXRpbmcgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoeCwgKiprd2FyZ3MpCgoKZGVmIHByZWRpY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIHJlc3VsdF9zZXQ6IE9wdGlvbmFsW3N0cl0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIFByZWRpY3RpbmcgZGF0YXNldCBieSBhIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIHByZWRpY3QgdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJLCBhIEZlYXR1cmVWZWN0b3Igb3IgYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgaW4gYSBzaGFwZSBvZiBhIGxpc3QvZGljdC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV2hlbiBwYXNzaW5nIGEgc2FtcGxlLCBwYXNzIHRoZSBkYXRhc2V0IGFzIGEgZmllbGQgaW4gYHBhcmFtc2AgaW5zdGVhZCBvZiBgaW5wdXRzYC4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHRoZSBkYXRhc2V0IGlzIGEgbGlzdC9kaWN0IHRoaXMgcGFyYW1ldGVyIHNob3VsZCBiZSByZXByZXNlbnRlZCBieSBpbnRlZ2Vycy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIHJlc3VsdF9zZXQ6ICAgICAgICAgICAgICBUaGUgZGIga2V5IHRvIHNldCBuYW1lIG9mIHRoZSBwcmVkaWN0aW9uIHJlc3VsdCBhbmQgdGhlIGZpbGVuYW1lLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0IHRvICdwcmVkaWN0aW9uJy4KICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgdG8gdGhlIHByZWRpY3QgZnVuY3Rpb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKFBSRURJQ1RfIHByZWZpeCBpcyBub3QgcmVxdWlyZWQpLgogICAgIiIiCiAgICAjIEdldCBkYXRhc2V0IGJ5IFVSTCBvciBieSBGZWF0dXJlVmVjdG9yOgogICAgZGF0YXNldCwgbGFiZWxfY29sdW1ucyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICBkYXRhc2V0PWRhdGFzZXQsCiAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICApCgogICAgIyBsb2FkaW5nIHRoZSBtb2RlbCwgYW5kIGdldHRpbmcgdGhlIG1vZGVsIGhhbmRsZXI6CiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbCwgY29udGV4dD1jb250ZXh0KQoKICAgICMgRHJvcHBpbmcgbGFiZWwgY29sdW1ucyBpZiBuZWNlc3Nhcnk6CiAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW10KICAgIGVsaWYgaXNpbnN0YW5jZShsYWJlbF9jb2x1bW5zLCBzdHIpOgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbbGFiZWxfY29sdW1uc10KCiAgICAjIFByZWRpY3Rpbmc6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibWFraW5nIHByZWRpY3Rpb24gYnkgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICB5X3ByZWQgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoZGF0YXNldCwgKiprd2FyZ3MpCgogICAgIyBQcmVwYXJpbmcgYW5kIHZhbGlkYXRpbmcgbGFiZWwgY29sdW1ucyBmb3IgdGhlIGRhdGFmcmFtZSBvZiB0aGUgcHJlZGljdGlvbiByZXN1bHQ6CiAgICBudW1fcHJlZGljdGVkID0gMSBpZiBsZW4oeV9wcmVkLnNoYXBlKSA9PSAxIGVsc2UgeV9wcmVkLnNoYXBlWzFdCgogICAgaWYgbnVtX3ByZWRpY3RlZCA+IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBpZiBudW1fcHJlZGljdGVkID09IDE6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbInByZWRpY3RlZCBsYWJlbHMiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMuZXh0ZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIGYicHJlZGljdGVkX2xhYmVsX3tpICsgMSArIGxlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgICAgICAgICAgICAgZm9yIGkgaW4gcmFuZ2UobnVtX3ByZWRpY3RlZCAtIGxlbihsYWJlbF9jb2x1bW5zKSkKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgZWxpZiBudW1fcHJlZGljdGVkIDwgbGVuKGxhYmVsX2NvbHVtbnMpOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIm51bWJlciBvZiBwcmVkaWN0ZWQgbGFiZWxzOiB7bnVtX3ByZWRpY3RlZH0gaXMgc21hbGxlciB0aGFuIG51bWJlciBvZiBsYWJlbCBjb2x1bW5zOiB7bGVuKGxhYmVsX2NvbHVtbnMpfSIKICAgICAgICApCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcgoKICAgIGFydGlmYWN0X25hbWUgPSByZXN1bHRfc2V0IG9yICJwcmVkaWN0aW9uIgogICAgbGFiZWxzX2luc2lkZV9kZiA9IHNldChsYWJlbF9jb2x1bW5zKSAmIHNldChkYXRhc2V0LmNvbHVtbnMudG9saXN0KCkpCiAgICBpZiBsYWJlbHNfaW5zaWRlX2RmOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIlRoZSBsYWJlbHM6IHtsYWJlbHNfaW5zaWRlX2RmfSBhcmUgYWxyZWFkeSBleGlzdGVkIGluIHRoZSBkYXRhZnJhbWUiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgIHByZWRfZGYgPSBwZC5jb25jYXQoW2RhdGFzZXQsIHBkLkRhdGFGcmFtZSh5X3ByZWQsIGNvbHVtbnM9bGFiZWxfY29sdW1ucyldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KGFydGlmYWN0X25hbWUsIHByZWRfZGYsIGRiX2tleT1yZXN1bHRfc2V0KQo= + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICBzdG9yZV91cmlfcHJlZml4LCBfID0gbWxydW4uZGF0YXN0b3JlLnBhcnNlX3N0b3JlX3VyaShkYXRhc2V0LmFydGlmYWN0X3VybCkKCiAgICAjIEdldHRpbmcgdGhlIGRhdGFzZXQ6CiAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgIyBGZWF0dXJlVmVjdG9yIGNhc2U6CiAgICAgICAgZnYgPSBtbHJ1bi5kYXRhc3RvcmUuZ2V0X3N0b3JlX3Jlc291cmNlKGRhdGFzZXQuYXJ0aWZhY3RfdXJsKQogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQoKICAgIGVsaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMgbm90IHByb3ZpZGVkLCBtYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBlbGlmIGlzaW5zdGFuY2UoZGF0YXNldCwgKGxpc3QsIGRpY3QpKToKICAgICAgICAjIGxpc3QvZGljdCBjYXNlOgogICAgICAgIGRhdGFzZXQgPSBwZC5EYXRhRnJhbWUoZGF0YXNldCkKICAgICAgICAjIENoZWNraW5nIGlmIGRyb3BfY29sdW1ucyBwcm92aWRlZCBieSBpbnRlZ2VyIHR5cGU6CiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgc3RyKSBvciAoCiAgICAgICAgICAgICAgICBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgbGlzdCkKICAgICAgICAgICAgICAgIGFuZCBhbnkoaXNpbnN0YW5jZShjb2wsIHN0cikgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpCiAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIG11c3QgYmUgYW4gaW50ZWdlci9saXN0IG9mIGludGVnZXJzIGlmIG5vdCBwcm92aWRlZCB3aXRoIGEgVVJJL0ZlYXR1cmVWZWN0b3IgZGF0YXNldCIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xLCBpbnBsYWNlPVRydWUpCgogICAgZWxzZToKICAgICAgICAjIHNpbXBsZSBVUkwgY2FzZToKICAgICAgICBkYXRhc2V0ID0gZGF0YXNldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgIm5vdCBhbGwgb2YgdGhlIGNvbHVtbnMgdG8gZHJvcCBpbiB0aGUgZGF0YXNldCwgZHJvcCBjb2x1bW5zIHByb2Nlc3Mgc2tpcHBlZCIKICAgICAgICAgICAgICAgICkKCiAgICByZXR1cm4gZGF0YXNldCwgbGFiZWxfY29sdW1ucwoKCmRlZiB0cmFpbigKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBtb2RlbF9jbGFzczogc3RyLAogICAgbGFiZWxfY29sdW1uczogT3B0aW9uYWxbVW5pb25bc3RyLCBMaXN0W3N0cl1dXSA9IE5vbmUsCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSAibW9kZWwiLAogICAgdGFnOiBzdHIgPSAiIiwKICAgIHNhbXBsZV9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRlc3Rfc2V0OiBEYXRhSXRlbSA9IE5vbmUsCiAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemU6IGZsb2F0ID0gTm9uZSwKICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gTm9uZSwKICAgIGxhYmVsczogZGljdCA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgVHJhaW5pbmcgYSBtb2RlbCB3aXRoIHRoZSBnaXZlbiBkYXRhc2V0LgoKICAgIGV4YW1wbGU6OgoKICAgICAgICBpbXBvcnQgbWxydW4KICAgICAgICBwcm9qZWN0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9wcm9qZWN0KCJteS1wcm9qZWN0IikKICAgICAgICBwcm9qZWN0LnNldF9mdW5jdGlvbigiaHViOi8vYXV0b190cmFpbmVyIiwgInRyYWluIikKICAgICAgICB0cmFpbmVyX3J1biA9IHByb2plY3QucnVuKAogICAgICAgICAgICBuYW1lPSJ0cmFpbiIsCiAgICAgICAgICAgIGhhbmRsZXI9InRyYWluIiwKICAgICAgICAgICAgaW5wdXRzPXsiZGF0YXNldCI6ICIuL3BhdGgvdG8vZGF0YXNldC5jc3YifSwKICAgICAgICAgICAgcGFyYW1zPXsKICAgICAgICAgICAgICAgICJtb2RlbF9jbGFzcyI6ICJza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb24iLAogICAgICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMiOiAibGFiZWwiLAogICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyI6ICJpZCIsCiAgICAgICAgICAgICAgICAibW9kZWxfbmFtZSI6ICJteS1tb2RlbCIsCiAgICAgICAgICAgICAgICAidGFnIjogInYxLjAuMCIsCiAgICAgICAgICAgICAgICAic2FtcGxlX3NldCI6ICIuL3BhdGgvdG8vc2FtcGxlX3NldC5jc3YiLAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IjogIi4vcGF0aC90by90ZXN0X3NldC5jc3YiLAogICAgICAgICAgICAgICAgIkNMQVNTX3NvbHZlciI6ICJsaWJsaW5lYXIiLAogICAgICAgICAgICB9LAogICAgICAgICkKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQKICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIG1vZGVsX2NsYXNzOiAgICAgICAgICAgICBUaGUgY2xhc3Mgb2YgdGhlIG1vZGVsLCBlLmcuIGBza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb25gCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uczogICAgICAgICAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuIE1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyIG9yIGEgbGlzdCBvZiBzdHJpbmdzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW5zIHRvIGRyb3AKICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgbmFtZSB0byB1c2UgZm9yIHN0b3JpbmcgdGhlIG1vZGVsIGFydGlmYWN0LCBkZWZhdWx0IHRvICdtb2RlbCcKICAgIDpwYXJhbSB0YWc6ICAgICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgdGFnIHRvIGxvZyB3aXRoCiAgICA6cGFyYW0gc2FtcGxlX3NldDogICAgICAgICAgICAgIEEgc2FtcGxlIHNldCBvZiBpbnB1dHMgZm9yIHRoZSBtb2RlbCBmb3IgbG9nZ2luZyBpdHMgc3RhdHMgYWxvbmcgdGhlIG1vZGVsIGluIGZhdm91cgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvZiBtb2RlbCBtb25pdG9yaW5nLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIHRlc3Rfc2V0OiAgICAgICAgICAgICAgICBUaGUgdGVzdCBzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIHdpdGguCiAgICA6cGFyYW0gdHJhaW5fdGVzdF9zcGxpdF9zaXplOiAgIGlmIHRlc3Rfc2V0IHdhcyBwcm92aWRlZCB0aGVuIHRoaXMgYXJndW1lbnQgaXMgaWdub3JlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgU2hvdWxkIGJlIGJldHdlZW4gMC4wIGFuZCAxLjAgYW5kIHJlcHJlc2VudCB0aGUgcHJvcG9ydGlvbiBvZiB0aGUgZGF0YXNldCB0byBpbmNsdWRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGluIHRoZSB0ZXN0IHNwbGl0LiBUaGUgc2l6ZSBvZiB0aGUgVHJhaW5pbmcgc2V0IGlzIHNldCB0byB0aGUgY29tcGxlbWVudCBvZiB0aGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHZhbHVlLiBEZWZhdWx0ID0gMC4yCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICAgICAgIFJlbGV2YW50IG9ubHkgd2hlbiB1c2luZyB0cmFpbl90ZXN0X3NwbGl0X3NpemUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgcmFuZG9tIHN0YXRlIHNlZWQgdG8gc2h1ZmZsZSB0aGUgZGF0YS4gRm9yIG1vcmUgaW5mb3JtYXRpb24sIHNlZToKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tX3N0YXRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE5vdGljZSB0aGF0IGhlcmUgd2Ugb25seSBwYXNzIGludGVnZXIgdmFsdWVzLgogICAgOnBhcmFtIGxhYmVsczogICAgICAgICAgICAgICAgICBMYWJlbHMgdG8gbG9nIHdpdGggdGhlIG1vZGVsCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHdpdGggcHJlZml4ZXMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoYXQgd2lsbCBiZSBwYXJzZWQgYW5kIHBhc3NlZCB0byB0aGUgcmVsZXZhbnQgZnVuY3Rpb24sIGJ5IHRoZSBmb2xsb3dpbmcgcHJlZml4ZXM6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYENMQVNTX2AgLSBmb3IgdGhlIG1vZGVsIGNsYXNzIGFyZ3VtZW50cwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIGBGSVRfYCAtIGZvciB0aGUgYGZpdGAgZnVuY3Rpb24gYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYFRSQUlOX2AgLSBmb3IgdGhlIGB0cmFpbmAgZnVuY3Rpb24gKGluIHhnYiBvciBsZ2JtIHRyYWluIGZ1bmN0aW9uIC0gZnV0dXJlKQoKICAgICIiIgogICAgIyBWYWxpZGF0ZSBpbnB1dHM6CiAgICAjIENoZWNrIGlmIGV4YWN0bHkgb25lIG9mIHRoZW0gaXMgc3VwcGxpZWQ6CiAgICBpZiB0ZXN0X3NldCBpcyBOb25lOgogICAgICAgIGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBpcyBOb25lOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IG9yIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBhcmUgbm90IHByb3ZpZGVkLCBzZXR0aW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB0byAwLjIiCiAgICAgICAgICAgICkKICAgICAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gMC4yCgogICAgZWxpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgInRlc3Rfc2V0IHByb3ZpZGVkLCBpZ25vcmluZyBnaXZlbiB0cmFpbl90ZXN0X3NwbGl0X3NpemUgdmFsdWUiCiAgICAgICAgKQogICAgICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSA9IE5vbmUKCiAgICAjIEdldCBEYXRhRnJhbWUgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIEdldHRpbmcgdGhlIHNhbXBsZSBzZXQ6CiAgICBpZiBzYW1wbGVfc2V0IGlzIE5vbmU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgZiJTYW1wbGUgc2V0IG5vdCBnaXZlbiwgdXNpbmcgdGhlIHdob2xlIHRyYWluaW5nIHNldCBhcyB0aGUgc2FtcGxlIHNldCIKICAgICAgICApCiAgICAgICAgc2FtcGxlX3NldCA9IGRhdGFzZXQKICAgIGVsc2U6CiAgICAgICAgc2FtcGxlX3NldCwgXyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIGRhdGFzZXQ9c2FtcGxlX3NldCwKICAgICAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgICAgICkKCiAgICAjIFBhcnNpbmcga3dhcmdzOgogICAgIyBUT0RPOiBVc2UgaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24uCiAgICB0cmFpbl9rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeChzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLlRSQUlOKQogICAgZml0X2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuRklUKQogICAgbW9kZWxfY2xhc3Nfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoCiAgICAgICAgc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5NT0RFTF9DTEFTUwogICAgKQoKICAgICMgQ2hlY2sgaWYgbW9kZWwgb3IgZnVuY3Rpb246CiAgICBpZiBoYXNhdHRyKG1vZGVsX2NsYXNzLCAidHJhaW4iKToKICAgICAgICAjIFRPRE86IE5lZWQgdG8gY2FsbDogbW9kZWwoKSwgYWZ0ZXJ3YXJkcyB0byBzdGFydCB0aGUgdHJhaW4gZnVuY3Rpb24uCiAgICAgICAgIyBtb2RlbCA9IGNyZWF0ZV9mdW5jdGlvbihmInttb2RlbF9jbGFzc30udHJhaW4iKQogICAgICAgIHJhaXNlIE5vdEltcGxlbWVudGVkRXJyb3IKICAgIGVsc2U6CiAgICAgICAgIyBDcmVhdGluZyBtb2RlbCBpbnN0YW5jZToKICAgICAgICBtb2RlbCA9IGNyZWF0ZV9jbGFzcyhtb2RlbF9jbGFzcykoKiptb2RlbF9jbGFzc19rd2FyZ3MpCgogICAgeCA9IGRhdGFzZXQuZHJvcChsYWJlbF9jb2x1bW5zLCBheGlzPTEpCiAgICB5ID0gZGF0YXNldFtsYWJlbF9jb2x1bW5zXQogICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplOgogICAgICAgIHhfdHJhaW4sIHhfdGVzdCwgeV90cmFpbiwgeV90ZXN0ID0gdHJhaW5fdGVzdF9zcGxpdCgKICAgICAgICAgICAgeCwgeSwgdGVzdF9zaXplPXRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSwgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZQogICAgICAgICkKICAgIGVsc2U6CiAgICAgICAgeF90cmFpbiwgeV90cmFpbiA9IHgsIHkKCiAgICAgICAgdGVzdF9zZXQgPSB0ZXN0X3NldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICB0ZXN0X3NldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKCiAgICAgICAgeF90ZXN0LCB5X3Rlc3QgPSB0ZXN0X3NldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSksIHRlc3Rfc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgQXV0b01MUnVuLmFwcGx5X21scnVuKAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIG1vZGVsX25hbWU9bW9kZWxfbmFtZSwKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgdGFnPXRhZywKICAgICAgICBzYW1wbGVfc2V0PXNhbXBsZV9zZXQsCiAgICAgICAgeV9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgdGVzdF9zZXQ9dGVzdF9zZXQsCiAgICAgICAgeF90ZXN0PXhfdGVzdCwKICAgICAgICB5X3Rlc3Q9eV90ZXN0LAogICAgICAgIGFydGlmYWN0cz1jb250ZXh0LmFydGlmYWN0cywKICAgICAgICBsYWJlbHM9bGFiZWxzLAogICAgKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInRyYWluaW5nICd7bW9kZWxfbmFtZX0nIikKICAgIG1vZGVsLmZpdCh4X3RyYWluLCB5X3RyYWluLCAqKmZpdF9rd2FyZ3MpCgoKZGVmIGV2YWx1YXRlKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBtb2RlbDogc3RyLAogICAgZGF0YXNldDogbWxydW4uRGF0YUl0ZW0sCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBFdmFsdWF0aW5nIGEgbW9kZWwuIEFydGlmYWN0cyBnZW5lcmF0ZWQgYnkgdGhlIE1MSGFuZGxlci4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgICAgICAgICAgIFRoZSBtb2RlbCBTdG9yZSBwYXRoLgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byBldmFsdWF0ZSB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcC4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIFBhcnNpbmcgbGFiZWxfY29sdW1uczoKICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zID0gW10KICAgIGlmIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9ICgKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyBpZiBpc2luc3RhbmNlKGxhYmVsX2NvbHVtbnMsIGxpc3QpIGVsc2UgW2xhYmVsX2NvbHVtbnNdCiAgICAgICAgKQogICAgICAgIGZvciBsYyBpbiBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBpZiBmcy5jb21tb24uZmVhdHVyZV9zZXBhcmF0b3IgaW4gbGM6CiAgICAgICAgICAgICAgICBmZWF0dXJlX3NldF9uYW1lLCBsYWJlbF9uYW1lLCBhbGlhcyA9IGZzLmNvbW1vbi5wYXJzZV9mZWF0dXJlX3N0cmluZyhsYykKICAgICAgICAgICAgICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zLmFwcGVuZChhbGlhcyBvciBsYWJlbF9uYW1lKQogICAgICAgIGlmIHBhcnNlZF9sYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gcGFyc2VkX2xhYmVsX2NvbHVtbnMKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgIyBMb2FkaW5nIHRoZSBtb2RlbCBhbmQgcHJlZGljdGluZzoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbCgKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQsIG1vZGVsX25hbWU9Im1vZGVsX0xpbmVhclJlZ3Jlc3Npb24iCiAgICApCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4obW9kZWxfaGFuZGxlci5tb2RlbCwgeV90ZXN0PXksIG1vZGVsX3BhdGg9bW9kZWwpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmImV2YWx1YXRpbmcgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoeCwgKiprd2FyZ3MpCgoKZGVmIHByZWRpY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIHJlc3VsdF9zZXQ6IE9wdGlvbmFsW3N0cl0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIFByZWRpY3RpbmcgZGF0YXNldCBieSBhIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIHByZWRpY3QgdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJLCBhIEZlYXR1cmVWZWN0b3Igb3IgYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgaW4gYSBzaGFwZSBvZiBhIGxpc3QvZGljdC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV2hlbiBwYXNzaW5nIGEgc2FtcGxlLCBwYXNzIHRoZSBkYXRhc2V0IGFzIGEgZmllbGQgaW4gYHBhcmFtc2AgaW5zdGVhZCBvZiBgaW5wdXRzYC4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHRoZSBkYXRhc2V0IGlzIGEgbGlzdC9kaWN0IHRoaXMgcGFyYW1ldGVyIHNob3VsZCBiZSByZXByZXNlbnRlZCBieSBpbnRlZ2Vycy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIHJlc3VsdF9zZXQ6ICAgICAgICAgICAgICBUaGUgZGIga2V5IHRvIHNldCBuYW1lIG9mIHRoZSBwcmVkaWN0aW9uIHJlc3VsdCBhbmQgdGhlIGZpbGVuYW1lLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0IHRvICdwcmVkaWN0aW9uJy4KICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgdG8gdGhlIHByZWRpY3QgZnVuY3Rpb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKFBSRURJQ1RfIHByZWZpeCBpcyBub3QgcmVxdWlyZWQpLgogICAgIiIiCiAgICAjIEdldCBkYXRhc2V0IGJ5IFVSTCBvciBieSBGZWF0dXJlVmVjdG9yOgogICAgZGF0YXNldCwgbGFiZWxfY29sdW1ucyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICBkYXRhc2V0PWRhdGFzZXQsCiAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICApCgogICAgIyBsb2FkaW5nIHRoZSBtb2RlbCwgYW5kIGdldHRpbmcgdGhlIG1vZGVsIGhhbmRsZXI6CiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbCwgY29udGV4dD1jb250ZXh0KQoKICAgICMgRHJvcHBpbmcgbGFiZWwgY29sdW1ucyBpZiBuZWNlc3Nhcnk6CiAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW10KICAgIGVsaWYgaXNpbnN0YW5jZShsYWJlbF9jb2x1bW5zLCBzdHIpOgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbbGFiZWxfY29sdW1uc10KCiAgICAjIFByZWRpY3Rpbmc6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibWFraW5nIHByZWRpY3Rpb24gYnkgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICB5X3ByZWQgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoZGF0YXNldCwgKiprd2FyZ3MpCgogICAgIyBQcmVwYXJpbmcgYW5kIHZhbGlkYXRpbmcgbGFiZWwgY29sdW1ucyBmb3IgdGhlIGRhdGFmcmFtZSBvZiB0aGUgcHJlZGljdGlvbiByZXN1bHQ6CiAgICBudW1fcHJlZGljdGVkID0gMSBpZiBsZW4oeV9wcmVkLnNoYXBlKSA9PSAxIGVsc2UgeV9wcmVkLnNoYXBlWzFdCgogICAgaWYgbnVtX3ByZWRpY3RlZCA+IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBpZiBudW1fcHJlZGljdGVkID09IDE6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbInByZWRpY3RlZCBsYWJlbHMiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMuZXh0ZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIGYicHJlZGljdGVkX2xhYmVsX3tpICsgMSArIGxlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgICAgICAgICAgICAgZm9yIGkgaW4gcmFuZ2UobnVtX3ByZWRpY3RlZCAtIGxlbihsYWJlbF9jb2x1bW5zKSkKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgZWxpZiBudW1fcHJlZGljdGVkIDwgbGVuKGxhYmVsX2NvbHVtbnMpOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIm51bWJlciBvZiBwcmVkaWN0ZWQgbGFiZWxzOiB7bnVtX3ByZWRpY3RlZH0gaXMgc21hbGxlciB0aGFuIG51bWJlciBvZiBsYWJlbCBjb2x1bW5zOiB7bGVuKGxhYmVsX2NvbHVtbnMpfSIKICAgICAgICApCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcgoKICAgIGFydGlmYWN0X25hbWUgPSByZXN1bHRfc2V0IG9yICJwcmVkaWN0aW9uIgogICAgbGFiZWxzX2luc2lkZV9kZiA9IHNldChsYWJlbF9jb2x1bW5zKSAmIHNldChkYXRhc2V0LmNvbHVtbnMudG9saXN0KCkpCiAgICBpZiBsYWJlbHNfaW5zaWRlX2RmOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIlRoZSBsYWJlbHM6IHtsYWJlbHNfaW5zaWRlX2RmfSBhcmUgYWxyZWFkeSBleGlzdGVkIGluIHRoZSBkYXRhZnJhbWUiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgIHByZWRfZGYgPSBwZC5jb25jYXQoW2RhdGFzZXQsIHBkLkRhdGFGcmFtZSh5X3ByZWQsIGNvbHVtbnM9bGFiZWxfY29sdW1ucyldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KGFydGlmYWN0X25hbWUsIHByZWRfZGYsIGRiX2tleT1yZXN1bHRfc2V0KQo= commands: [] code_origin: '' origin_filename: '' @@ -86,7 +86,7 @@ spec: doc: Labels to log with the model default: null outputs: [] - lineno: 117 + lineno: 116 has_varargs: false has_kwargs: true evaluate: @@ -112,7 +112,7 @@ spec: Classification tasks. Mandatory when dataset is not a FeatureVector. default: null outputs: [] - lineno: 269 + lineno: 268 has_varargs: false has_kwargs: true predict: @@ -147,7 +147,7 @@ spec: to 'prediction'. default: null outputs: [] - lineno: 323 + lineno: 322 has_varargs: false has_kwargs: true description: Automatic train, evaluate and predict functions for the ML frameworks diff --git a/auto_trainer/item.yaml b/auto_trainer/item.yaml index 122c68048..ffa03bf0c 100755 --- a/auto_trainer/item.yaml +++ b/auto_trainer/item.yaml @@ -23,4 +23,4 @@ spec: kind: job requirements: [] url: '' -version: 1.6.0 +version: 1.7.0