diff --git a/neuronxcc-2.16.345.0+69131dd3/0_REGISTRY/0.0.28.dev0/training/llama/michaelbenayoun/llama-2-tiny-4kv-heads-4layers-random/e4b7f2d4b9fdeed1fc72.json b/neuronxcc-2.16.345.0+69131dd3/0_REGISTRY/0.0.28.dev0/training/llama/michaelbenayoun/llama-2-tiny-4kv-heads-4layers-random/e4b7f2d4b9fdeed1fc72.json new file mode 100644 index 0000000000000000000000000000000000000000..7b48ae9f88384d56410dbb645c3ae8760d69a271 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/0_REGISTRY/0.0.28.dev0/training/llama/michaelbenayoun/llama-2-tiny-4kv-heads-4layers-random/e4b7f2d4b9fdeed1fc72.json @@ -0,0 +1 @@ +{"_attn_implementation_autoset": true, "architectures": ["LlamaForCausalLM"], "attention_bias": false, "attention_dropout": 0.0, "bos_token_id": 1, "eos_token_id": 2, "head_dim": 4, "hidden_act": "silu", "hidden_size": 128, "initializer_range": 0.02, "intermediate_size": 128, "is_decoder": true, "max_position_embeddings": 4096, "mlp_bias": false, "model_type": "llama", "neuron": {"compiler_version": "2.16.345.0+69131dd3", "input_specs": {"attention_mask": [1, 128], "input_ids": [1, 128], "labels": [1, 128]}, "model_class": "LlamaForCausalLM", "num_neuron_cores_per_node": 2, "pipeline_parallel_size": 1, "precision": "float32", "tensor_parallel_size": 1, "training": true}, "num_attention_heads": 32, "num_hidden_layers": 4, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000.0, "tie_word_embeddings": false, "torch_dtype": "float32", "use_cache": true, "vocab_size": 32000} \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..2dc5ec11bba81035d6b74baaee21c637d69e6dbd --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:741e0b7e1cd35d27b3848f3974a0e2d479de880e8965741c0b9d8c0f02b5e3b5 +size 1848 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..987b8ed063ae98ff8b170d05bdd19d6900119360 Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..28233f668299e4f06ad971c6018d1725e8b87880 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4b81ff56bfc39354a840db1955192322509af7d679c975d8184d15a6e0fe5e0 +size 299 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..0bf4ff201d743fcb0a6b1c1415a610f4769d71aa Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..32157f229be17480afc9e02a7ed48ca0cbc8f22a --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..275dc780fefc32e68a6cc70e866395f3b12d974b --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f79c2d87a0ddcf0b26820282a1a90e805a69b92404101a0963689d7e4308927b +size 121417 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..e1e4ca683948d5674384f7a2d2e6ac188f510937 Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..990c3966faf5c5d184f2362257d11b490a67c1b7 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1bc054c5f5ede2a2051ce74e48f607791b85eba3a90d541637b29079cbc87d2 +size 148332 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/model.log b/neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/model.log new file mode 100644 index 0000000000000000000000000000000000000000..f7684541b3136ef1dd4829127ba1b4ed9261c486 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/model.log @@ -0,0 +1,57 @@ +Failed compilation with ['neuronx-cc', 'compile', '--framework=XLA', '/tmp/no-user/neuroncc_compile_workdir/c010323a-c1cc-4661-9aa1-91df76d78037/model.MODULE_12445205521823827895+3cc9a3cb.hlo_module.pb', '--output', '/tmp/no-user/neuroncc_compile_workdir/c010323a-c1cc-4661-9aa1-91df76d78037/model.MODULE_12445205521823827895+3cc9a3cb.neff', '--target=trn1', '--model-type=transformer', '--verbose=35']: Process Process-1: +Traceback (most recent call last): + File "neuronxcc/driver/CommandDriver.py", line 345, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand + File "neuronxcc/driver/commands/CompileCommand.py", line 1353, in neuronxcc.driver.commands.CompileCommand.CompileCommand.run + File "neuronxcc/driver/commands/CompileCommand.py", line 1304, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline + File "neuronxcc/driver/commands/CompileCommand.py", line 1324, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline + File "neuronxcc/driver/commands/CompileCommand.py", line 1327, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline + File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run + File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState + File "neuronxcc/driver/Pipeline.py", line 30, in neuronxcc.driver.Pipeline.Pipeline.runSingleInput + File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run + File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState + File "neuronxcc/driver/jobs/Frontend.py", line 454, in neuronxcc.driver.jobs.Frontend.Frontend.runSingleInput + File "neuronxcc/driver/jobs/Frontend.py", line 218, in neuronxcc.driver.jobs.Frontend.Frontend.runXLAFrontend + File "neuronxcc/driver/jobs/Frontend.py", line 190, in neuronxcc.driver.jobs.Frontend.Frontend.runHlo2Tensorizer +neuronxcc.driver.Exceptions.CompilerInvalidInputException: ERROR: Failed command /home/runner/_work/optimum-neuron/optimum-neuron/aws_neuron_venv_pytorch/lib/python3.10/site-packages/neuronxcc/starfish/bin/hlo2penguin --input /tmp/no-user/neuroncc_compile_workdir/c010323a-c1cc-4661-9aa1-91df76d78037/model.MODULE_12445205521823827895+3cc9a3cb.hlo_module.pb --out-dir ./ --output penguin.py --remat --max-costly-ops=2 --max-live-in-size=5 --max-remat-chain-size=10 --max-mem-multiple=1.8 --min-def-use-distance=500 --remat-policy=transformer --allow-same-pass-remat=true --split-abc --layers-per-module=1 --emit-tensor-level-dropout-ops --emit-tensor-level-rng-ops --native-to-custom-softmax --partitioner-opts='--transformer' +------------ +Reported stdout: +DEBUG: needsModular? No. macCnt 0 +INFO: Switching to single-module compile. PrePartitionPipe skipped. +INFO: Found memory bound graph +INFO: Number of Native SoftmaxDx's detected and replaced: 0 +INFO: Number of Native Softmax's detected and replaced: 0 +Replaced 0 dropout sequences with OffloadedDropout +INFO: HloMacCount has found 0 +INFO: Traffic has found 68311702 +INFO: AIF 0 +HLO Ops used in computation: add all-reduce broadcast compare constant copy divide get-tuple-element multiply parameter power reduce reshape select sqrt subtract tuple +2025-01-07 14:52:23.605725: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %tuple.1 = tuple(%add.806, %copy.119, %add.902, %add.950, %copy.122, %add.1046, %copy.124, %copy.125, %copy.126, %copy.127, %copy.128, %copy.129, %copy.130, %copy.131, %copy.132, %copy.133, %copy.134, %copy.135, %copy.136, %copy.137, %copy.138, %copy.139, %copy.140, %copy.141, %copy.142, %copy.143, %copy.144, %copy.145, %copy.146, %copy.147, %add.2245, %copy.149, %copy.150, %copy.151, %copy.152, %copy.153, %copy.154, %copy.155, %copy.156, %constant.122, %copy.158, %copy.159, %copy.160, %copy.161, %copy.1... to 512 characters in the compiler's debug metadata +Invoking RemoveOptimizationBarriers pass +2025-01-07 14:52:23.618811: F hilo/MLIRPasses/Transforms/VerifyAliasing.cc:125] false && "Read after write on aliased input/output buffer." + +------------ +Reported stderr: +None +------------ +Import of the HLO graph into the Neuron Compiler has failed. +This may be caused by unsupported operators or an internal compiler error. +More details can be found in the error message(s) above. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "neuronxcc/driver/CommandDriver.py", line 352, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand_in_process + File "neuronxcc/driver/CommandDriver.py", line 347, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand + File "neuronxcc/driver/CommandDriver.py", line 111, in neuronxcc.driver.CommandDriver.handleError + File "neuronxcc/driver/GlobalState.py", line 102, in neuronxcc.driver.GlobalState.FinalizeGlobalState + File "neuronxcc/driver/GlobalState.py", line 82, in neuronxcc.driver.GlobalState._GlobalStateImpl.shutdown + File "/usr/lib/python3.10/shutil.py", line 715, in rmtree + onerror(os.lstat, path, sys.exc_info()) + File "/usr/lib/python3.10/shutil.py", line 713, in rmtree + orig_st = os.lstat(path) +FileNotFoundError: [Errno 2] No such file or directory: '/tmp/no-user/neuroncc_compile_workdir/c010323a-c1cc-4661-9aa1-91df76d78037/neuronxcc-9f3m6nhk' diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..1abf8c1f5b08427297ce8721f35f5f4c00ebdcce --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a67865aabfb27a273f32eb2a882f6269447d64eb1e7a72583db03ddd45d87c1 +size 301 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..97b7f5851391681082193b56f336fa1611faf7de Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..7d199d2b1303506dbd6d658db6aff4121f673863 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:113f044831a4e1c32f50de67f9bac777f9c1f5ce03532c84565e3c20b6d0c473 +size 141306 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/model.log b/neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/model.log new file mode 100644 index 0000000000000000000000000000000000000000..121f12bef98e3e5aad992699b0ae1ef9142365a8 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/model.log @@ -0,0 +1,59 @@ +Failed compilation with ['neuronx-cc', 'compile', '--framework=XLA', '/tmp/no-user/neuroncc_compile_workdir/9a13359e-4fac-4cca-91ac-310afbe0c188/model.MODULE_14466467378454838209+3cc9a3cb.hlo_module.pb', '--output', '/tmp/no-user/neuroncc_compile_workdir/9a13359e-4fac-4cca-91ac-310afbe0c188/model.MODULE_14466467378454838209+3cc9a3cb.neff', '--target=trn1', '--model-type=transformer', '--verbose=35']: Process Process-1: +Traceback (most recent call last): + File "neuronxcc/driver/CommandDriver.py", line 345, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand + File "neuronxcc/driver/commands/CompileCommand.py", line 1353, in neuronxcc.driver.commands.CompileCommand.CompileCommand.run + File "neuronxcc/driver/commands/CompileCommand.py", line 1304, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline + File "neuronxcc/driver/commands/CompileCommand.py", line 1324, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline + File "neuronxcc/driver/commands/CompileCommand.py", line 1327, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline + File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run + File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState + File "neuronxcc/driver/Pipeline.py", line 30, in neuronxcc.driver.Pipeline.Pipeline.runSingleInput + File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run + File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState + File "neuronxcc/driver/jobs/Frontend.py", line 454, in neuronxcc.driver.jobs.Frontend.Frontend.runSingleInput + File "neuronxcc/driver/jobs/Frontend.py", line 218, in neuronxcc.driver.jobs.Frontend.Frontend.runXLAFrontend + File "neuronxcc/driver/jobs/Frontend.py", line 190, in neuronxcc.driver.jobs.Frontend.Frontend.runHlo2Tensorizer +neuronxcc.driver.Exceptions.CompilerInvalidInputException: ERROR: Failed command /home/runner/_work/optimum-neuron/optimum-neuron/aws_neuron_venv_pytorch/lib/python3.10/site-packages/neuronxcc/starfish/bin/hlo2penguin --input /tmp/no-user/neuroncc_compile_workdir/9a13359e-4fac-4cca-91ac-310afbe0c188/model.MODULE_14466467378454838209+3cc9a3cb.hlo_module.pb --out-dir ./ --output penguin.py --remat --max-costly-ops=2 --max-live-in-size=5 --max-remat-chain-size=10 --max-mem-multiple=1.8 --min-def-use-distance=500 --remat-policy=transformer --allow-same-pass-remat=true --split-abc --layers-per-module=1 --emit-tensor-level-dropout-ops --emit-tensor-level-rng-ops --native-to-custom-softmax --partitioner-opts='--transformer' +------------ +Reported stdout: +DEBUG: needsModular? No. macCnt 0 +INFO: Switching to single-module compile. PrePartitionPipe skipped. +INFO: Found memory bound graph +INFO: Number of Native SoftmaxDx's detected and replaced: 0 +INFO: Number of Native Softmax's detected and replaced: 0 +Replaced 0 dropout sequences with OffloadedDropout +INFO: HloMacCount has found 0 +INFO: Traffic has found 170744446 +INFO: AIF 0 +HLO Ops used in computation: add all-reduce broadcast compare concatenate constant copy divide get-tuple-element multiply parameter reduce reshape select sqrt subtract tuple +2025-01-07 14:48:59.478813: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %concatenate.439 = concatenate(%reshape.400, %reshape.401, %reshape.402, %reshape.403, %reshape.404, %reshape.405, %reshape.406, %reshape.407, %reshape.408, %reshape.409, %reshape.410, %reshape.411, %reshape.412, %reshape.413, %reshape.414, %reshape.415, %reshape.416, %reshape.417, %reshape.418, %reshape.419, %reshape.420, %reshape.421, %reshape.422, %reshape.423, %reshape.424, %reshape.425, %reshape.426, %reshape.427, %reshape.428, %reshape.429, %reshape.430, %reshape.431, %reshape.432, %reshape.433, %r... to 512 characters in the compiler's debug metadata +2025-01-07 14:48:59.479598: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %all-reduce.580 = all-reduce(%multiply.534, %multiply.532, %multiply.530, %multiply.528, %multiply.526, %multiply.524, %multiply.522, %multiply.520, %multiply.518, %multiply.516, %multiply.514, %multiply.512, %multiply.510, %multiply.508, %multiply.506, %multiply.504, %multiply.502, %multiply.500, %multiply.498, %multiply.496, %multiply.494, %multiply.492, %multiply.490, %multiply.488, %multiply.486, %multiply.484, %multiply.482, %multiply.480, %multiply.478, %multiply.476, %multiply.474, %multiply.472, ... to 512 characters in the compiler's debug metadata +2025-01-07 14:48:59.479761: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %tuple.1 = tuple(%add.774, %add.808, %add.842, %copy.121, %copy.122, %copy.123, %copy.124, %copy.125, %add.1046, %copy.127, %copy.128, %copy.129, %copy.130, %copy.131, %copy.132, %copy.133, %copy.134, %copy.135, %copy.136, %copy.137, %copy.138, %copy.139, %copy.140, %copy.141, %copy.142, %copy.143, %copy.144, %copy.145, %copy.146, %copy.147, %copy.148, %copy.149, %copy.150, %copy.151, %copy.152, %copy.153, %copy.154, %copy.155, %copy.156, %constant.12, %copy.158, %copy.159, %copy.160, %copy.161, %copy.16... to 512 characters in the compiler's debug metadata +Invoking RemoveOptimizationBarriers pass +2025-01-07 14:48:59.492367: F hilo/MLIRPasses/Transforms/VerifyAliasing.cc:125] false && "Read after write on aliased input/output buffer." + +------------ +Reported stderr: +None +------------ +Import of the HLO graph into the Neuron Compiler has failed. +This may be caused by unsupported operators or an internal compiler error. +More details can be found in the error message(s) above. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "neuronxcc/driver/CommandDriver.py", line 352, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand_in_process + File "neuronxcc/driver/CommandDriver.py", line 347, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand + File "neuronxcc/driver/CommandDriver.py", line 111, in neuronxcc.driver.CommandDriver.handleError + File "neuronxcc/driver/GlobalState.py", line 102, in neuronxcc.driver.GlobalState.FinalizeGlobalState + File "neuronxcc/driver/GlobalState.py", line 82, in neuronxcc.driver.GlobalState._GlobalStateImpl.shutdown + File "/usr/lib/python3.10/shutil.py", line 715, in rmtree + onerror(os.lstat, path, sys.exc_info()) + File "/usr/lib/python3.10/shutil.py", line 713, in rmtree + orig_st = os.lstat(path) +FileNotFoundError: [Errno 2] No such file or directory: '/tmp/no-user/neuroncc_compile_workdir/9a13359e-4fac-4cca-91ac-310afbe0c188/neuronxcc-i3pdyt1j' diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..2d765435d694c8c725d604d49b2338858027921f --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b83fbb315057e53957c7f172e7d7a5deeda3955f7c8625082ad4b2c0a55a5a6a +size 118841 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..cab33295662a060a91b5308a7ba8b2a978a5c778 Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..32157f229be17480afc9e02a7ed48ca0cbc8f22a --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..007cc4123fbcd78ae106afdb279d2b953a47859d --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0afb122b426ab077b2d37431b30bc1bff549e04be1b825219b2eeb0973ffa46 +size 1792 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..dc2dd8c3619a2e79b605384b63a8032e542e74dd Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..5eaaa7f04cd0173c70d68972b7831f6009a761d6 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4385afb5b7af6963fa2374ea47ac8eb2ec073c8242b564f970bc65cd8a13eab1 +size 126812 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..ea7342164b2e39a34e4703cfadbc77991662a5fc Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..d13db79b1c043f5d27aed56b0071c31e1dc606e8 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58bcd7af24a3d214372af69d0fbefa4e192482762d9e75b0dab4cdb7a5144964 +size 151549 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/model.log b/neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/model.log new file mode 100644 index 0000000000000000000000000000000000000000..2c473662eeae8f9d1552e75227e649d5d52a86bc --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/model.log @@ -0,0 +1,59 @@ +Failed compilation with ['neuronx-cc', 'compile', '--framework=XLA', '/tmp/no-user/neuroncc_compile_workdir/eb8c59e7-8e37-4ec2-80c9-43abb7d60b2c/model.MODULE_1590662263737562288+3cc9a3cb.hlo_module.pb', '--output', '/tmp/no-user/neuroncc_compile_workdir/eb8c59e7-8e37-4ec2-80c9-43abb7d60b2c/model.MODULE_1590662263737562288+3cc9a3cb.neff', '--target=trn1', '--model-type=transformer', '--verbose=35']: Process Process-1: +Traceback (most recent call last): + File "neuronxcc/driver/CommandDriver.py", line 345, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand + File "neuronxcc/driver/commands/CompileCommand.py", line 1353, in neuronxcc.driver.commands.CompileCommand.CompileCommand.run + File "neuronxcc/driver/commands/CompileCommand.py", line 1304, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline + File "neuronxcc/driver/commands/CompileCommand.py", line 1324, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline + File "neuronxcc/driver/commands/CompileCommand.py", line 1327, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline + File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run + File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState + File "neuronxcc/driver/Pipeline.py", line 30, in neuronxcc.driver.Pipeline.Pipeline.runSingleInput + File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run + File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState + File "neuronxcc/driver/jobs/Frontend.py", line 454, in neuronxcc.driver.jobs.Frontend.Frontend.runSingleInput + File "neuronxcc/driver/jobs/Frontend.py", line 218, in neuronxcc.driver.jobs.Frontend.Frontend.runXLAFrontend + File "neuronxcc/driver/jobs/Frontend.py", line 190, in neuronxcc.driver.jobs.Frontend.Frontend.runHlo2Tensorizer +neuronxcc.driver.Exceptions.CompilerInvalidInputException: ERROR: Failed command /home/runner/_work/optimum-neuron/optimum-neuron/aws_neuron_venv_pytorch/lib/python3.10/site-packages/neuronxcc/starfish/bin/hlo2penguin --input /tmp/no-user/neuroncc_compile_workdir/eb8c59e7-8e37-4ec2-80c9-43abb7d60b2c/model.MODULE_1590662263737562288+3cc9a3cb.hlo_module.pb --out-dir ./ --output penguin.py --remat --max-costly-ops=2 --max-live-in-size=5 --max-remat-chain-size=10 --max-mem-multiple=1.8 --min-def-use-distance=500 --remat-policy=transformer --allow-same-pass-remat=true --split-abc --layers-per-module=1 --emit-tensor-level-dropout-ops --emit-tensor-level-rng-ops --native-to-custom-softmax --partitioner-opts='--transformer' +------------ +Reported stdout: +DEBUG: needsModular? No. macCnt 0 +INFO: Switching to single-module compile. PrePartitionPipe skipped. +INFO: Found memory bound graph +INFO: Number of Native SoftmaxDx's detected and replaced: 0 +INFO: Number of Native Softmax's detected and replaced: 0 +Replaced 0 dropout sequences with OffloadedDropout +INFO: HloMacCount has found 0 +INFO: Traffic has found 182891677 +INFO: AIF 0 +HLO Ops used in computation: add all-reduce broadcast compare concatenate constant copy divide get-tuple-element multiply parameter reduce reshape select sqrt subtract tuple +2025-01-07 14:44:53.575264: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %concatenate.674 = concatenate(%reshape.633, %reshape.634, %reshape.635, %reshape.636, %reshape.637, %reshape.638, %reshape.639, %reshape.640, %reshape.641, %reshape.642, %reshape.643, %reshape.644, %reshape.645, %reshape.646, %reshape.647, %reshape.648, %reshape.649, %reshape.650, %reshape.651, %reshape.652, %reshape.653, %reshape.654, %reshape.655, %reshape.656, %reshape.657, %reshape.658, %reshape.659, %reshape.660, %reshape.661, %reshape.662, %reshape.663, %reshape.664, %reshape.665, %reshape.666, %r... to 512 characters in the compiler's debug metadata +2025-01-07 14:44:53.575951: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %all-reduce.803 = all-reduce(%multiply.761, %multiply.759, %multiply.757, %multiply.755, %multiply.753, %multiply.751, %multiply.749, %multiply.747, %multiply.745, %multiply.743, %multiply.741, %multiply.739, %multiply.737, %multiply.735, %multiply.733, %multiply.731, %multiply.729, %multiply.727, %multiply.725, %multiply.723, %multiply.721, %multiply.719, %multiply.717, %multiply.715, %multiply.713, %multiply.711, %multiply.709, %multiply.707, %multiply.705, %multiply.703, %multiply.701, %multiply.699, ... to 512 characters in the compiler's debug metadata +2025-01-07 14:44:53.575971: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %tuple.1 = tuple(%add.1039, %add.1081, %add.1123, %add.1165, %add.1207, %add.1249, %add.1291, %add.1333, %add.1375, %add.1417, %add.1459, %add.1501, %add.1543, %add.1585, %add.1627, %add.1669, %add.1711, %add.1753, %add.1795, %add.1837, %add.1879, %add.1921, %add.1963, %add.2005, %add.2047, %add.2089, %add.2131, %add.2173, %add.2215, %add.2257, %add.2299, %add.2341, %add.2383, %add.2425, %add.2467, %add.2509, %add.2551, %add.2593, %add.2635, %add.2677, %add.2719, %constant.265, %sqrt.683, %add.1024, %add... to 512 characters in the compiler's debug metadata +Invoking RemoveOptimizationBarriers pass +2025-01-07 14:44:53.587275: F hilo/MLIRPasses/Transforms/VerifyAliasing.cc:125] false && "Read after write on aliased input/output buffer." + +------------ +Reported stderr: +None +------------ +Import of the HLO graph into the Neuron Compiler has failed. +This may be caused by unsupported operators or an internal compiler error. +More details can be found in the error message(s) above. + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "neuronxcc/driver/CommandDriver.py", line 352, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand_in_process + File "neuronxcc/driver/CommandDriver.py", line 347, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand + File "neuronxcc/driver/CommandDriver.py", line 111, in neuronxcc.driver.CommandDriver.handleError + File "neuronxcc/driver/GlobalState.py", line 102, in neuronxcc.driver.GlobalState.FinalizeGlobalState + File "neuronxcc/driver/GlobalState.py", line 82, in neuronxcc.driver.GlobalState._GlobalStateImpl.shutdown + File "/usr/lib/python3.10/shutil.py", line 715, in rmtree + onerror(os.lstat, path, sys.exc_info()) + File "/usr/lib/python3.10/shutil.py", line 713, in rmtree + orig_st = os.lstat(path) +FileNotFoundError: [Errno 2] No such file or directory: '/tmp/no-user/neuroncc_compile_workdir/eb8c59e7-8e37-4ec2-80c9-43abb7d60b2c/neuronxcc-94ew1sgw' diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..080d1f59b48b98b31dc0b0d8a42d8a4b8f80ad4e --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8088b253d25f4710353508b7082528feb00434a6a1aea3e90767cce42e662b04 +size 138992 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..05efdab64e213012acc1c3bde21a1bb0eae588ba Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..32157f229be17480afc9e02a7ed48ca0cbc8f22a --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..33b8795ef60b0127f5a3addb983ee320c37f521b --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7590a741d6e1160f42eff0d8598ab24c5481dfe0c1256867ee875e6753df91a +size 122498 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..b151244a16d814c8873dafe9f1e48baaad43791f Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..dd5556505c4c9a2d2506a41bc6d13dd3f6a58a5d --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:152764326e8a64a6ea303060d3c1faf961b75b60082da6743a7cd4bbab6f456d +size 170704 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..26bc08b7bfe0594d6be1e5894bf9ab5ec8760003 Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..c1a6c8e47ad46b375031fba434297c7e27a89526 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f604d1cf6fc7e491741c3d9e304eb5e90943fb1b2de04f160237485628f85611 +size 34602 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..795f7aaec4d3b3d5b1c119c6473b49b10f339e48 Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_17726044688659330421+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..95b3be4845f8a82ba5ed862a8bf1c58c3bdfc77a --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ecb3d325f0ad71c5debfc335d6855d7193961adcbef234a447c95a2637256f9 +size 73494 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..28fb55992bf84fe03911d7e8b4c4d60796860fee Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_1791163358840385770+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..81fa4c6d6d3be419f00ca24f003537942828f7ba --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3401c7775a3be8dac9610d1a7ee18e0a317817e0afe32f02659a980fe7ec4d3 +size 146243 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..cf859d5fb0d72aa52ec39879b4a49bdb160d2f50 Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_18234269709931122811+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..563fd72427bc70f3bf07720d8c4e34f535c69da4 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9498d91605cf86ddb7b6b19dfc2127c76dd9ad12d9dcf1a25f452924a3735540 +size 120225 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..48de32a7ff6164d8955b7e343230f2a54af9d566 Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_2775021859393882420+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..7eb5a44344e2ad7b8025262d5ee8c811bee66bde --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64a5bf05daf7fc25addde961f9cc28e7045cb938d9c1e0eea79b5ee839a9d8e5 +size 140500 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..bd21209bc1c00b1beb7f1202861c1cef691141b0 Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_5116233657793453184+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..fbe3741203d493c70ebcf10e46a75fca547afbbd --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fa3a9ee8b48cba9d88f3199d9c9f10e9eef489ece36e6063f3a04c686554cec +size 171068 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..4a323b0a1b96933da381dfdebb11d6cb9be207a3 Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_5756301259811216641+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..65059d5e923a32e05c32687e896baf55309c77a2 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91d1f0fc428d9b928484af74c82b96b6502ae9902d8286c89f1f90ee77134ed7 +size 767 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..7883add2075df196982df721a34f6a3a0e832d64 Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_6167168901329397720+3cc9a3cb/model.neff differ diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/compile_flags.json b/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/compile_flags.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b6dc7253ea633264165599c24cf4686fef31 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/compile_flags.json @@ -0,0 +1 @@ +["--target=trn1", "--model-type=transformer"] \ No newline at end of file diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/model.done b/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/model.done new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/model.hlo_module.pb b/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/model.hlo_module.pb new file mode 100644 index 0000000000000000000000000000000000000000..415c67fdb45b87f627dd153d559b75d0ea73aa96 --- /dev/null +++ b/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/model.hlo_module.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d39467e75b746ca84d618c35f609f038bc50cb2e1c285c92e1c58243ca01e67 +size 147682 diff --git a/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/model.neff b/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/model.neff new file mode 100644 index 0000000000000000000000000000000000000000..bb6e00af55f20c0c1be08d4300e3632ef5321eca Binary files /dev/null and b/neuronxcc-2.16.345.0+69131dd3/MODULE_8341285887588434315+3cc9a3cb/model.neff differ