chris/MiniGPT-4
1
0
Fork 0
mirror of https://github.com/Vision-CAIR/MiniGPT-4.git synced 2025-04-05 18:40:46 +00:00
Code Issues Packages Projects Releases Wiki Activity
5bec4d0608
MiniGPT-4/Post_Route_Universal_PromptMoE_RawProb_backward_graph

6291 lines
242 KiB
Plaintext
Raw Normal View History

adjust universal experts & datasets
2024-03-28 06:48:42 +00:00
digraph {
graph [size="885.0,885.0"]
node [align=left fontname=monospace fontsize=10 height=0.2 ranksep=0.1 shape=box style=filled]
140193037219536 [label="
(4, 45, 768)" fillcolor=darkolivegreen1]
140193039136752 [label=CatBackward0]
140193570151248 -> 140193039136752
140193570151248 [label=AddBackward0]
140193039092752 -> 140193570151248
140193039092752 [label=IndexBackward0]
140193578428064 -> 140193039092752
140193578428064 [label=NativeLayerNormBackward0]
140193578427440 -> 140193578428064
140193578427440 [label=AddBackward0]
140193578427632 -> 140193578427440
140193578427632 [label=CatBackward0]
140193578427584 -> 140193578427632
140193578427584 [label=CatBackward0]
140193578428016 -> 140193578427584
140193578428016 [label=SliceBackward0]
140193578428304 -> 140193578428016
140193578428304 [label=SliceBackward0]
140193578427248 -> 140193578428304
140193578427248 [label=SliceBackward0]
140193578427152 -> 140193578427248
140193578427152 [label=SumBackward1]
140193578427056 -> 140193578427152
140193578427056 [label=MulBackward0]
140193578426960 -> 140193578427056
140193578426960 [label=IndexBackward0]
140193578428160 -> 140193578426960
140193578428160 [label=PermuteBackward0]
140193578428256 -> 140193578428160
140193578428256 [label=CatBackward0]
140193578428352 -> 140193578428256
140193578428352 [label=UnsqueezeBackward0]
140193578428688 -> 140193578428352
140193578428688 [label=NativeDropoutBackward0]
140193578428784 -> 140193578428688
140193578428784 [label=ViewBackward0]
140193578428880 -> 140193578428784
140193578428880 [label=AddmmBackward0]
140193578428976 -> 140193578428880
140193578428976 [label=ToCopyBackward0]
140193578429168 -> 140193578428976
140193039388000 [label="encoder.layer.11.experts.experts.0.dense2.bias
(768)" fillcolor=lightblue]
140193039388000 -> 140193578429168
140193578429168 [label=AccumulateGrad]
140193578428736 -> 140193578428880
140193578428736 [label=ViewBackward0]
140193578429024 -> 140193578428736
140193578429024 [label=GeluBackward0]
140193578429120 -> 140193578429024
140193578429120 [label=ViewBackward0]
140193578429216 -> 140193578429120
140193578429216 [label=AddmmBackward0]
140193578429312 -> 140193578429216
140193578429312 [label=ToCopyBackward0]
140193578429504 -> 140193578429312
140193039388320 [label="encoder.layer.11.experts.experts.0.dense1.bias
(3072)" fillcolor=lightblue]
140193039388320 -> 140193578429504
140193578429504 [label=AccumulateGrad]
140193578429456 -> 140193578429216
140193578429456 [label=ViewBackward0]
140193578429744 -> 140193578429456
140193578429744 [label=ToCopyBackward0]
140193578429840 -> 140193578429744
140193578429840 [label=IndexBackward0]
140193578427392 -> 140193578429840
140193578427392 [label=SliceBackward0]
140193578429792 -> 140193578427392
140193578429792 [label=SliceBackward0]
140193578429888 -> 140193578429792
140193578429888 [label=SliceBackward0]
140193578429984 -> 140193578429888
140193578429984 [label=SliceBackward0]
140193578430080 -> 140193578429984
140193578430080 [label=SliceBackward0]
140193578430176 -> 140193578430080
140193578430176 [label=NativeLayerNormBackward0]
140193578430272 -> 140193578430176
140193578430272 [label=AddBackward0]
140193036800064 -> 140193578430272
140193036800064 [label=NativeDropoutBackward0]
140193036800400 -> 140193036800064
140193036800400 [label=ViewBackward0]
140193036800496 -> 140193036800400
140193036800496 [label=AddmmBackward0]
140193036800592 -> 140193036800496
140193036800592 [label=ToCopyBackward0]
140193036800784 -> 140193036800592
140193039417648 [label="encoder.layer.11.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039417648 -> 140193036800784
140193036800784 [label=AccumulateGrad]
140193036800352 -> 140193036800496
140193036800352 [label=ViewBackward0]
140193036800640 -> 140193036800352
140193036800640 [label=ViewBackward0]
140193036800736 -> 140193036800640
140193036800736 [label=CloneBackward0]
140193036800832 -> 140193036800736
140193036800832 [label=PermuteBackward0]
140193036800928 -> 140193036800832
140193036800928 [label=UnsafeViewBackward0]
140193036801024 -> 140193036800928
140193036801024 [label=BmmBackward0]
140193036801120 -> 140193036801024
140193036801120 [label=ReshapeAliasBackward0]
140193036801456 -> 140193036801120
140193036801456 [label=ExpandBackward0]
140193036801552 -> 140193036801456
140193036801552 [label=ToCopyBackward0]
140193036801648 -> 140193036801552
140193036801648 [label=NativeDropoutBackward0]
140193036801744 -> 140193036801648
140193036801744 [label=SoftmaxBackward0]
140193036801840 -> 140193036801744
140193036801840 [label=AddBackward0]
140193036801936 -> 140193036801840
140193036801936 [label=DivBackward0]
140193036802032 -> 140193036801936
140193036802032 [label=UnsafeViewBackward0]
140193036802128 -> 140193036802032
140193036802128 [label=BmmBackward0]
140193036802224 -> 140193036802128
140193036802224 [label=UnsafeViewBackward0]
140193036802176 -> 140193036802224
140193036802176 [label=CloneBackward0]
140193036802272 -> 140193036802176
140193036802272 [label=ExpandBackward0]
140193036802368 -> 140193036802272
140193036802368 [label=PermuteBackward0]
140193036802464 -> 140193036802368
140193036802464 [label=ViewBackward0]
140193036802560 -> 140193036802464
140193036802560 [label=ViewBackward0]
140193036802656 -> 140193036802560
140193036802656 [label=AddmmBackward0]
140193036802752 -> 140193036802656
140193036802752 [label=ToCopyBackward0]
140193036802944 -> 140193036802752
140193039418608 [label="encoder.layer.11.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039418608 -> 140193036802944
140193036802944 [label=AccumulateGrad]
140193036802896 -> 140193036802656
140193036802896 [label=ViewBackward0]
140193036803184 -> 140193036802896
140193036803184 [label=ToCopyBackward0]
140193036800208 -> 140193036803184
140193036800208 [label=CatBackward0]
140193036803136 -> 140193036800208
140193036803136 [label=NativeLayerNormBackward0]
140193036803472 -> 140193036803136
140193036803472 [label=AddBackward0]
140193036803664 -> 140193036803472
140193036803664 [label=CatBackward0]
140193036803616 -> 140193036803664
140193036803616 [label=CatBackward0]
140193036804048 -> 140193036803616
140193036804048 [label=SliceBackward0]
140193036803904 -> 140193036804048
140193036803904 [label=SliceBackward0]
140193036849360 -> 140193036803904
140193036849360 [label=SliceBackward0]
140193036849456 -> 140193036849360
140193036849456 [label=SumBackward1]
140193036849552 -> 140193036849456
140193036849552 [label=MulBackward0]
140193036849648 -> 140193036849552
140193036849648 [label=IndexBackward0]
140193036849792 -> 140193036849648
140193036849792 [label=PermuteBackward0]
140193036849888 -> 140193036849792
140193036849888 [label=CatBackward0]
140193036849984 -> 140193036849888
140193036849984 [label=UnsqueezeBackward0]
140193036850128 -> 140193036849984
140193036850128 [label=NativeDropoutBackward0]
140193036850224 -> 140193036850128
140193036850224 [label=ViewBackward0]
140193036850320 -> 140193036850224
140193036850320 [label=AddmmBackward0]
140193036850416 -> 140193036850320
140193036850416 [label=ToCopyBackward0]
140193036850608 -> 140193036850416
140193039404464 [label="encoder.layer.10.experts.experts.0.dense2.bias
(768)" fillcolor=lightblue]
140193039404464 -> 140193036850608
140193036850608 [label=AccumulateGrad]
140193036850368 -> 140193036850320
140193036850368 [label=ViewBackward0]
140193036850656 -> 140193036850368
140193036850656 [label=GeluBackward0]
140193036850752 -> 140193036850656
140193036850752 [label=ViewBackward0]
140193036850848 -> 140193036850752
140193036850848 [label=AddmmBackward0]
140193036850944 -> 140193036850848
140193036850944 [label=ToCopyBackward0]
140193036851136 -> 140193036850944
140193039404704 [label="encoder.layer.10.experts.experts.0.dense1.bias
(3072)" fillcolor=lightblue]
140193039404704 -> 140193036851136
140193036851136 [label=AccumulateGrad]
140193036850896 -> 140193036850848
140193036850896 [label=ViewBackward0]
140193036851184 -> 140193036850896
140193036851184 [label=ToCopyBackward0]
140193036851280 -> 140193036851184
140193036851280 [label=IndexBackward0]
140193036803424 -> 140193036851280
140193036803424 [label=SliceBackward0]
140193036851424 -> 140193036803424
140193036851424 [label=SliceBackward0]
140193036851520 -> 140193036851424
140193036851520 [label=NativeLayerNormBackward0]
140193036851616 -> 140193036851520
140193036851616 [label=AddBackward0]
140193036851808 -> 140193036851616
140193036851808 [label=NativeDropoutBackward0]
140193036851952 -> 140193036851808
140193036851952 [label=ViewBackward0]
140193036852048 -> 140193036851952
140193036852048 [label=AddmmBackward0]
140193036852144 -> 140193036852048
140193036852144 [label=ToCopyBackward0]
140193036852336 -> 140193036852144
140193039420448 [label="encoder.layer.10.crossattention.output.dense.bias
(768)" fillcolor=lightblue]
140193039420448 -> 140193036852336
140193036852336 [label=AccumulateGrad]
140193036852096 -> 140193036852048
140193036852096 [label=ViewBackward0]
140193036852528 -> 140193036852096
140193036852528 [label=ViewBackward0]
140193036852624 -> 140193036852528
140193036852624 [label=CloneBackward0]
140193036852816 -> 140193036852624
140193036852816 [label=PermuteBackward0]
140193036853008 -> 140193036852816
140193036853008 [label=UnsafeViewBackward0]
140193036853104 -> 140193036853008
140193036853104 [label=BmmBackward0]
140193036852960 -> 140193036853104
140193036852960 [label=ReshapeAliasBackward0]
140193036890320 -> 140193036852960
140193036890320 [label=ExpandBackward0]
140193036890368 -> 140193036890320
140193036890368 [label=ToCopyBackward0]
140193036890608 -> 140193036890368
140193036890608 [label=NativeDropoutBackward0]
140193036890800 -> 140193036890608
140193036890800 [label=SoftmaxBackward0]
140193036890848 -> 140193036890800
140193036890848 [label=AddBackward0]
140193036891088 -> 140193036890848
140193036891088 [label=DivBackward0]
140193036891280 -> 140193036891088
140193036891280 [label=UnsafeViewBackward0]
140193036891328 -> 140193036891280
140193036891328 [label=BmmBackward0]
140193036891568 -> 140193036891328
140193036891568 [label=UnsafeViewBackward0]
140193036891952 -> 140193036891568
140193036891952 [label=CloneBackward0]
140193036892144 -> 140193036891952
140193036892144 [label=ExpandBackward0]
140193036892336 -> 140193036892144
140193036892336 [label=PermuteBackward0]
140193036892432 -> 140193036892336
140193036892432 [label=ViewBackward0]
140193036892624 -> 140193036892432
140193036892624 [label=ViewBackward0]
140193036892816 -> 140193036892624
140193036892816 [label=AddmmBackward0]
140193036892912 -> 140193036892816
140193036892912 [label=ToCopyBackward0]
140193036893296 -> 140193036892912
140193039421168 [label="encoder.layer.10.crossattention.self.query.bias
(768)" fillcolor=lightblue]
140193039421168 -> 140193036893296
140193036893296 [label=AccumulateGrad]
140193036892720 -> 140193036892816
140193036892720 [label=ViewBackward0]
140193036893200 -> 140193036892720
140193036893200 [label=ToCopyBackward0]
140193036851760 -> 140193036893200
140193036851760 [label=SliceBackward0]
140193036893584 -> 140193036851760
140193036893584 [label=SliceBackward0]
140193036893776 -> 140193036893584
140193036893776 [label=SliceBackward0]
140193036893872 -> 140193036893776
140193036893872 [label=NativeLayerNormBackward0]
140193036894064 -> 140193036893872
140193036894064 [label=AddBackward0]
140193036914848 -> 140193036894064
140193036914848 [label=NativeDropoutBackward0]
140193036914992 -> 140193036914848
140193036914992 [label=ViewBackward0]
140193036915184 -> 140193036914992
140193036915184 [label=AddmmBackward0]
140193036915232 -> 140193036915184
140193036915232 [label=ToCopyBackward0]
140193036915664 -> 140193036915232
140193039429776 [label="encoder.layer.10.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039429776 -> 140193036915664
140193036915664 [label=AccumulateGrad]
140193036915376 -> 140193036915184
140193036915376 [label=ViewBackward0]
140193036915856 -> 140193036915376
140193036915856 [label=ViewBackward0]
140193036916048 -> 140193036915856
140193036916048 [label=CloneBackward0]
140193036916240 -> 140193036916048
140193036916240 [label=PermuteBackward0]
140193036916336 -> 140193036916240
140193036916336 [label=UnsafeViewBackward0]
140193036916528 -> 140193036916336
140193036916528 [label=BmmBackward0]
140193036916720 -> 140193036916528
140193036916720 [label=ReshapeAliasBackward0]
140193036916672 -> 140193036916720
140193036916672 [label=ExpandBackward0]
140193036916912 -> 140193036916672
140193036916912 [label=ToCopyBackward0]
140193036917104 -> 140193036916912
140193036917104 [label=NativeDropoutBackward0]
140193036917152 -> 140193036917104
140193036917152 [label=SoftmaxBackward0]
140193036917392 -> 140193036917152
140193036917392 [label=AddBackward0]
140193036917584 -> 140193036917392
140193036917584 [label=DivBackward0]
140193036917632 -> 140193036917584
140193036917632 [label=UnsafeViewBackward0]
140193036917872 -> 140193036917632
140193036917872 [label=BmmBackward0]
140193036918064 -> 140193036917872
140193036918064 [label=UnsafeViewBackward0]
140193036918448 -> 140193036918064
140193036918448 [label=CloneBackward0]
140193036918640 -> 140193036918448
140193036918640 [label=ExpandBackward0]
140193036918736 -> 140193036918640
140193036918736 [label=PermuteBackward0]
140193036918592 -> 140193036918736
140193036918592 [label=ViewBackward0]
140193036947856 -> 140193036918592
140193036947856 [label=ViewBackward0]
140193036947952 -> 140193036947856
140193036947952 [label=AddmmBackward0]
140193036948144 -> 140193036947952
140193036948144 [label=ToCopyBackward0]
140193036948432 -> 140193036948144
140193039432336 [label="encoder.layer.10.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039432336 -> 140193036948432
140193036948432 [label=AccumulateGrad]
140193036947808 -> 140193036947952
140193036947808 [label=ViewBackward0]
140193036948288 -> 140193036947808
140193036948288 [label=ToCopyBackward0]
140193036914800 -> 140193036948288
140193036914800 [label=CatBackward0]
140193036948816 -> 140193036914800
140193036948816 [label=NativeLayerNormBackward0]
140193036948768 -> 140193036948816
140193036948768 [label=AddBackward0]
140193036949200 -> 140193036948768
140193036949200 [label=CatBackward0]
140193036949584 -> 140193036949200
140193036949584 [label=CatBackward0]
140193036949728 -> 140193036949584
140193036949728 [label=SliceBackward0]
140193036950256 -> 140193036949728
140193036950256 [label=SliceBackward0]
140193036950352 -> 140193036950256
140193036950352 [label=SliceBackward0]
140193036950544 -> 140193036950352
140193036950544 [label=SumBackward1]
140193036950736 -> 140193036950544
140193036950736 [label=MulBackward0]
140193036950832 -> 140193036950736
140193036950832 [label=IndexBackward0]
140193036950928 -> 140193036950832
140193036950928 [label=PermuteBackward0]
140193036951120 -> 140193036950928
140193036951120 [label=CatBackward0]
140193036951168 -> 140193036951120
140193036951168 [label=UnsqueezeBackward0]
140193036951408 -> 140193036951168
140193036951408 [label=NativeDropoutBackward0]
140193036972336 -> 140193036951408
140193036972336 [label=ViewBackward0]
140193036972528 -> 140193036972336
140193036972528 [label=AddmmBackward0]
140193036972720 -> 140193036972528
140193036972720 [label=ToCopyBackward0]
140193036973008 -> 140193036972720
140193039431136 [label="encoder.layer.9.experts.experts.0.dense2.bias
(768)" fillcolor=lightblue]
140193039431136 -> 140193036973008
140193036973008 [label=AccumulateGrad]
140193036972432 -> 140193036972528
140193036972432 [label=ViewBackward0]
140193036972912 -> 140193036972432
140193036972912 [label=GeluBackward0]
140193036973104 -> 140193036972912
140193036973104 [label=ViewBackward0]
140193036973152 -> 140193036973104
140193036973152 [label=AddmmBackward0]
140193036973392 -> 140193036973152
140193036973392 [label=ToCopyBackward0]
140193036973632 -> 140193036973392
140193039431456 [label="encoder.layer.9.experts.experts.0.dense1.bias
(3072)" fillcolor=lightblue]
140193039431456 -> 140193036973632
140193036973632 [label=AccumulateGrad]
140193036973488 -> 140193036973152
140193036973488 [label=ViewBackward0]
140193036973968 -> 140193036973488
140193036973968 [label=ToCopyBackward0]
140193036974160 -> 140193036973968
140193036974160 [label=IndexBackward0]
140193036949296 -> 140193036974160
140193036949296 [label=SliceBackward0]
140193036974112 -> 140193036949296
140193036974112 [label=SliceBackward0]
140193036974352 -> 140193036974112
140193036974352 [label=SliceBackward0]
140193036974544 -> 140193036974352
140193036974544 [label=SliceBackward0]
140193036974592 -> 140193036974544
140193036974592 [label=SliceBackward0]
140193036974832 -> 140193036974592
140193036974832 [label=NativeLayerNormBackward0]
140193036975024 -> 140193036974832
140193036975024 [label=AddBackward0]
140193036975312 -> 140193036975024
140193036975312 [label=NativeDropoutBackward0]
140193036975696 -> 140193036975312
140193036975696 [label=ViewBackward0]
140193036975888 -> 140193036975696
140193036975888 [label=AddmmBackward0]
140193036976080 -> 140193036975888
140193036976080 [label=ToCopyBackward0]
140193036480816 -> 140193036976080
140193039442304 [label="encoder.layer.9.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039442304 -> 140193036480816
140193036480816 [label=AccumulateGrad]
140193036975792 -> 140193036975888
140193036975792 [label=ViewBackward0]
140193036480720 -> 140193036975792
140193036480720 [label=ViewBackward0]
140193036480912 -> 140193036480720
140193036480912 [label=CloneBackward0]
140193036480960 -> 140193036480912
140193036480960 [label=PermuteBackward0]
140193036481200 -> 140193036480960
140193036481200 [label=UnsafeViewBackward0]
140193036481392 -> 140193036481200
140193036481392 [label=BmmBackward0]
140193036481440 -> 140193036481392
140193036481440 [label=ReshapeAliasBackward0]
140193036481968 -> 140193036481440
140193036481968 [label=ExpandBackward0]
140193036482064 -> 140193036481968
140193036482064 [label=ToCopyBackward0]
140193036482256 -> 140193036482064
140193036482256 [label=NativeDropoutBackward0]
140193036482448 -> 140193036482256
140193036482448 [label=SoftmaxBackward0]
140193036482544 -> 140193036482448
140193036482544 [label=AddBackward0]
140193036482736 -> 140193036482544
140193036482736 [label=DivBackward0]
140193036482928 -> 140193036482736
140193036482928 [label=UnsafeViewBackward0]
140193036483024 -> 140193036482928
140193036483024 [label=BmmBackward0]
140193036483216 -> 140193036483024
140193036483216 [label=UnsafeViewBackward0]
140193036483312 -> 140193036483216
140193036483312 [label=CloneBackward0]
140193036483360 -> 140193036483312
140193036483360 [label=ExpandBackward0]
140193036483600 -> 140193036483360
140193036483600 [label=PermuteBackward0]
140193036483792 -> 140193036483600
140193036483792 [label=ViewBackward0]
140193036483840 -> 140193036483792
140193036483840 [label=ViewBackward0]
140193036484080 -> 140193036483840
140193036484080 [label=AddmmBackward0]
140193036484272 -> 140193036484080
140193036484272 [label=ToCopyBackward0]
140193036484320 -> 140193036484272
140193039445264 [label="encoder.layer.9.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039445264 -> 140193036484320
140193036484320 [label=AccumulateGrad]
140193036484368 -> 140193036484080
140193036484368 [label=ViewBackward0]
140193036513584 -> 140193036484368
140193036513584 [label=ToCopyBackward0]
140193036975408 -> 140193036513584
140193036975408 [label=CatBackward0]
140193036513536 -> 140193036975408
140193036513536 [label=NativeLayerNormBackward0]
140193036513968 -> 140193036513536
140193036513968 [label=AddBackward0]
140193036514256 -> 140193036513968
140193036514256 [label=CatBackward0]
140193036514640 -> 140193036514256
140193036514640 [label=CatBackward0]
140193036514928 -> 140193036514640
140193036514928 [label=SliceBackward0]
140193036515312 -> 140193036514928
140193036515312 [label=SliceBackward0]
140193036515504 -> 140193036515312
140193036515504 [label=SliceBackward0]
140193036515600 -> 140193036515504
140193036515600 [label=SumBackward1]
140193036515792 -> 140193036515600
140193036515792 [label=MulBackward0]
140193036515984 -> 140193036515792
140193036515984 [label=IndexBackward0]
140193036515936 -> 140193036515984
140193036515936 [label=PermuteBackward0]
140193036516176 -> 140193036515936
140193036516176 [label=CatBackward0]
140193036516368 -> 140193036516176
140193036516368 [label=UnsqueezeBackward0]
140193036516752 -> 140193036516368
140193036516752 [label=NativeDropoutBackward0]
140193036516944 -> 140193036516752
140193036516944 [label=ViewBackward0]
140193036517040 -> 140193036516944
140193036517040 [label=AddmmBackward0]
140193036517232 -> 140193036517040
140193036517232 [label=ToCopyBackward0]
140193036533920 -> 140193036517232
140193039444064 [label="encoder.layer.8.experts.experts.0.dense2.bias
(768)" fillcolor=lightblue]
140193039444064 -> 140193036533920
140193036533920 [label=AccumulateGrad]
140193036516896 -> 140193036517040
140193036516896 [label=ViewBackward0]
140193036533968 -> 140193036516896
140193036533968 [label=GeluBackward0]
140193036534064 -> 140193036533968
140193036534064 [label=ViewBackward0]
140193036534256 -> 140193036534064
140193036534256 [label=AddmmBackward0]
140193036534304 -> 140193036534256
140193036534304 [label=ToCopyBackward0]
140193036534736 -> 140193036534304
140193039443984 [label="encoder.layer.8.experts.experts.0.dense1.bias
(3072)" fillcolor=lightblue]
140193039443984 -> 140193036534736
140193036534736 [label=AccumulateGrad]
140193036534448 -> 140193036534256
140193036534448 [label=ViewBackward0]
140193036534928 -> 140193036534448
140193036534928 [label=ToCopyBackward0]
140193036535120 -> 140193036534928
140193036535120 [label=IndexBackward0]
140193036514352 -> 140193036535120
140193036514352 [label=SliceBackward0]
140193036535216 -> 140193036514352
140193036535216 [label=SliceBackward0]
140193036535264 -> 140193036535216
140193036535264 [label=NativeLayerNormBackward0]
140193036535504 -> 140193036535264
140193036535504 [label=AddBackward0]
140193036535744 -> 140193036535504
140193036535744 [label=NativeDropoutBackward0]
140193036536272 -> 140193036535744
140193036536272 [label=ViewBackward0]
140193036536368 -> 140193036536272
140193036536368 [label=AddmmBackward0]
140193036536560 -> 140193036536368
140193036536560 [label=ToCopyBackward0]
140193036536848 -> 140193036536560
140193039459488 [label="encoder.layer.8.crossattention.output.dense.bias
(768)" fillcolor=lightblue]
140193039459488 -> 140193036536848
140193036536848 [label=AccumulateGrad]
140193036536224 -> 140193036536368
140193036536224 [label=ViewBackward0]
140193036536704 -> 140193036536224
140193036536704 [label=ViewBackward0]
140193036536944 -> 140193036536704
140193036536944 [label=CloneBackward0]
140193036537136 -> 140193036536944
140193036537136 [label=PermuteBackward0]
140193036537184 -> 140193036537136
140193036537184 [label=UnsafeViewBackward0]
140193036537424 -> 140193036537184
140193036537424 [label=BmmBackward0]
140193036537616 -> 140193036537424
140193036537616 [label=ReshapeAliasBackward0]
140193036537664 -> 140193036537616
140193036537664 [label=ExpandBackward0]
140193036566928 -> 140193036537664
140193036566928 [label=ToCopyBackward0]
140193036567024 -> 140193036566928
140193036567024 [label=NativeDropoutBackward0]
140193036567216 -> 140193036567024
140193036567216 [label=SoftmaxBackward0]
140193036567408 -> 140193036567216
140193036567408 [label=AddBackward0]
140193036567504 -> 140193036567408
140193036567504 [label=DivBackward0]
140193036567696 -> 140193036567504
140193036567696 [label=UnsafeViewBackward0]
140193036567888 -> 140193036567696
140193036567888 [label=BmmBackward0]
140193036567984 -> 140193036567888
140193036567984 [label=UnsafeViewBackward0]
140193036568080 -> 140193036567984
140193036568080 [label=CloneBackward0]
140193036568272 -> 140193036568080
140193036568272 [label=ExpandBackward0]
140193036568320 -> 140193036568272
140193036568320 [label=PermuteBackward0]
140193036568560 -> 140193036568320
140193036568560 [label=ViewBackward0]
140193036568752 -> 140193036568560
140193036568752 [label=ViewBackward0]
140193036568800 -> 140193036568752
140193036568800 [label=AddmmBackward0]
140193036569040 -> 140193036568800
140193036569040 [label=ToCopyBackward0]
140193036569280 -> 140193036569040
140193039460208 [label="encoder.layer.8.crossattention.self.query.bias
(768)" fillcolor=lightblue]
140193039460208 -> 140193036569280
140193036569280 [label=AccumulateGrad]
140193036569136 -> 140193036568800
140193036569136 [label=ViewBackward0]
140193036569616 -> 140193036569136
140193036569616 [label=ToCopyBackward0]
140193036535888 -> 140193036569616
140193036535888 [label=SliceBackward0]
140193036569712 -> 140193036535888
140193036569712 [label=SliceBackward0]
140193036569760 -> 140193036569712
140193036569760 [label=SliceBackward0]
140193036570000 -> 140193036569760
140193036570000 [label=NativeLayerNormBackward0]
140193036570192 -> 140193036570000
140193036570192 [label=AddBackward0]
140193036570480 -> 140193036570192
140193036570480 [label=NativeDropoutBackward0]
140193036591408 -> 140193036570480
140193036591408 [label=ViewBackward0]
140193036591600 -> 140193036591408
140193036591600 [label=AddmmBackward0]
140193036591792 -> 140193036591600
140193036591792 [label=ToCopyBackward0]
140193036592080 -> 140193036591792
140193039460528 [label="encoder.layer.8.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039460528 -> 140193036592080
140193036592080 [label=AccumulateGrad]
140193036591504 -> 140193036591600
140193036591504 [label=ViewBackward0]
140193036591984 -> 140193036591504
140193036591984 [label=ViewBackward0]
140193036592176 -> 140193036591984
140193036592176 [label=CloneBackward0]
140193036592224 -> 140193036592176
140193036592224 [label=PermuteBackward0]
140193036592464 -> 140193036592224
140193036592464 [label=UnsafeViewBackward0]
140193036592656 -> 140193036592464
140193036592656 [label=BmmBackward0]
140193036592704 -> 140193036592656
140193036592704 [label=ReshapeAliasBackward0]
140193036593232 -> 140193036592704
140193036593232 [label=ExpandBackward0]
140193036593328 -> 140193036593232
140193036593328 [label=ToCopyBackward0]
140193036593520 -> 140193036593328
140193036593520 [label=NativeDropoutBackward0]
140193036593712 -> 140193036593520
140193036593712 [label=SoftmaxBackward0]
140193036593808 -> 140193036593712
140193036593808 [label=AddBackward0]
140193036594000 -> 140193036593808
140193036594000 [label=DivBackward0]
140193036594192 -> 140193036594000
140193036594192 [label=UnsafeViewBackward0]
140193036594288 -> 140193036594192
140193036594288 [label=BmmBackward0]
140193036594480 -> 140193036594288
140193036594480 [label=UnsafeViewBackward0]
140193036594576 -> 140193036594480
140193036594576 [label=CloneBackward0]
140193036594624 -> 140193036594576
140193036594624 [label=ExpandBackward0]
140193036594864 -> 140193036594624
140193036594864 [label=PermuteBackward0]
140193036595056 -> 140193036594864
140193036595056 [label=ViewBackward0]
140193036594384 -> 140193036595056
140193036594384 [label=ViewBackward0]
140193036628176 -> 140193036594384
140193036628176 [label=AddmmBackward0]
140193036628368 -> 140193036628176
140193036628368 [label=ToCopyBackward0]
140193036628656 -> 140193036628368
140193039467280 [label="encoder.layer.8.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039467280 -> 140193036628656
140193036628656 [label=AccumulateGrad]
140193036628464 -> 140193036628176
140193036628464 [label=ViewBackward0]
140193036628944 -> 140193036628464
140193036628944 [label=ToCopyBackward0]
140193036570576 -> 140193036628944
140193036570576 [label=CatBackward0]
140193036628896 -> 140193036570576
140193036628896 [label=NativeLayerNormBackward0]
140193036629424 -> 140193036628896
140193036629424 [label=AddBackward0]
140193036629712 -> 140193036629424
140193036629712 [label=CatBackward0]
140193036629808 -> 140193036629712
140193036629808 [label=CatBackward0]
140193036630384 -> 140193036629808
140193036630384 [label=SliceBackward0]
140193036630336 -> 140193036630384
140193036630336 [label=SliceBackward0]
140193036630576 -> 140193036630336
140193036630576 [label=SliceBackward0]
140193036630768 -> 140193036630576
140193036630768 [label=SumBackward1]
140193036630816 -> 140193036630768
140193036630816 [label=MulBackward0]
140193036631056 -> 140193036630816
140193036631056 [label=IndexBackward0]
140193036631440 -> 140193036631056
140193036631440 [label=PermuteBackward0]
140193036631632 -> 140193036631440
140193036631632 [label=CatBackward0]
140193036631824 -> 140193036631632
140193036631824 [label=UnsqueezeBackward0]
140193036631776 -> 140193036631824
140193036631776 [label=NativeDropoutBackward0]
140193036631728 -> 140193036631776
140193036631728 [label=ViewBackward0]
140193036656848 -> 140193036631728
140193036656848 [label=AddmmBackward0]
140193036656896 -> 140193036656848
140193036656896 [label=ToCopyBackward0]
140193036657328 -> 140193036656896
140193039461888 [label="encoder.layer.7.experts.experts.0.dense2.bias
(768)" fillcolor=lightblue]
140193039461888 -> 140193036657328
140193036657328 [label=AccumulateGrad]
140193036657040 -> 140193036656848
140193036657040 [label=ViewBackward0]
140193036657520 -> 140193036657040
140193036657520 [label=GeluBackward0]
140193036657712 -> 140193036657520
140193036657712 [label=ViewBackward0]
140193036657904 -> 140193036657712
140193036657904 [label=AddmmBackward0]
140193036658000 -> 140193036657904
140193036658000 [label=ToCopyBackward0]
140193036658384 -> 140193036658000
140193039462208 [label="encoder.layer.7.experts.experts.0.dense1.bias
(3072)" fillcolor=lightblue]
140193039462208 -> 140193036658384
140193036658384 [label=AccumulateGrad]
140193036657808 -> 140193036657904
140193036657808 [label=ViewBackward0]
140193036658288 -> 140193036657808
140193036658288 [label=ToCopyBackward0]
140193036658336 -> 140193036658288
140193036658336 [label=IndexBackward0]
140193036629376 -> 140193036658336
140193036629376 [label=SliceBackward0]
140193036658864 -> 140193036629376
140193036658864 [label=SliceBackward0]
140193036658960 -> 140193036658864
140193036658960 [label=SliceBackward0]
140193036659152 -> 140193036658960
140193036659152 [label=SliceBackward0]
140193036659344 -> 140193036659152
140193036659344 [label=SliceBackward0]
140193036659440 -> 140193036659344
140193036659440 [label=NativeLayerNormBackward0]
140193036659632 -> 140193036659440
140193036659632 [label=AddBackward0]
140193036659920 -> 140193036659632
140193036659920 [label=NativeDropoutBackward0]
140193036660016 -> 140193036659920
140193036660016 [label=ViewBackward0]
140193036660208 -> 140193036660016
140193036660208 [label=AddmmBackward0]
140193036660256 -> 140193036660208
140193036660256 [label=ToCopyBackward0]
140193036660496 -> 140193036660256
140193039487200 [label="encoder.layer.7.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039487200 -> 140193036660496
140193036660496 [label=AccumulateGrad]
140193036660400 -> 140193036660208
140193036660400 [label=ViewBackward0]
140193036677280 -> 140193036660400
140193036677280 [label=ViewBackward0]
140193036677520 -> 140193036677280
140193036677520 [label=CloneBackward0]
140193036677712 -> 140193036677520
140193036677712 [label=PermuteBackward0]
140193036677808 -> 140193036677712
140193036677808 [label=UnsafeViewBackward0]
140193036678000 -> 140193036677808
140193036678000 [label=BmmBackward0]
140193036678192 -> 140193036678000
140193036678192 [label=ReshapeAliasBackward0]
140193036678144 -> 140193036678192
140193036678144 [label=ExpandBackward0]
140193036678384 -> 140193036678144
140193036678384 [label=ToCopyBackward0]
140193036678576 -> 140193036678384
140193036678576 [label=NativeDropoutBackward0]
140193036678624 -> 140193036678576
140193036678624 [label=SoftmaxBackward0]
140193036678864 -> 140193036678624
140193036678864 [label=AddBackward0]
140193036679056 -> 140193036678864
140193036679056 [label=DivBackward0]
140193036679104 -> 140193036679056
140193036679104 [label=UnsafeViewBackward0]
140193036679344 -> 140193036679104
140193036679344 [label=BmmBackward0]
140193036679536 -> 140193036679344
140193036679536 [label=UnsafeViewBackward0]
140193036679920 -> 140193036679536
140193036679920 [label=CloneBackward0]
140193036680112 -> 140193036679920
140193036680112 [label=ExpandBackward0]
140193036680208 -> 140193036680112
140193036680208 [label=PermuteBackward0]
140193036680400 -> 140193036680208
140193036680400 [label=ViewBackward0]
140193036680592 -> 140193036680400
140193036680592 [label=ViewBackward0]
140193036680688 -> 140193036680592
140193036680688 [label=AddmmBackward0]
140193036680880 -> 140193036680688
140193036680880 [label=ToCopyBackward0]
140193036681168 -> 140193036680880
140193039488080 [label="encoder.layer.7.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039488080 -> 140193036681168
140193036681168 [label=AccumulateGrad]
140193036680544 -> 140193036680688
140193036680544 [label=ViewBackward0]
140193036681072 -> 140193036680544
140193036681072 [label=ToCopyBackward0]
140193036659728 -> 140193036681072
140193036659728 [label=CatBackward0]
140193036710288 -> 140193036659728
140193036710288 [label=NativeLayerNormBackward0]
140193036710240 -> 140193036710288
140193036710240 [label=AddBackward0]
140193036710672 -> 140193036710240
140193036710672 [label=CatBackward0]
140193036711056 -> 140193036710672
140193036711056 [label=CatBackward0]
140193036711200 -> 140193036711056
140193036711200 [label=SliceBackward0]
140193036711728 -> 140193036711200
140193036711728 [label=SliceBackward0]
140193036711824 -> 140193036711728
140193036711824 [label=SliceBackward0]
140193036712016 -> 140193036711824
140193036712016 [label=SumBackward1]
140193036712208 -> 140193036712016
140193036712208 [label=MulBackward0]
140193036712304 -> 140193036712208
140193036712304 [label=IndexBackward0]
140193036712400 -> 140193036712304
140193036712400 [label=PermuteBackward0]
140193036712592 -> 140193036712400
140193036712592 [label=ViewBackward0]
140193036712640 -> 140193036712592
140193036712640 [label=CloneBackward0]
140193036712880 -> 140193036712640
140193036712880 [label=ExpandBackward0]
140193036713072 -> 140193036712880
140193036713072 [label=UnsqueezeBackward0]
140193036713120 -> 140193036713072
140193036713120 [label=CatBackward0]
140193036713360 -> 140193036713120
140193036713360 [label=UnsqueezeBackward0]
140193036713744 -> 140193036713360
140193036713744 [label=NativeDropoutBackward0]
140193036713936 -> 140193036713744
140193036713936 [label=ViewBackward0]
140193036713840 -> 140193036713936
140193036713840 [label=AddmmBackward0]
140193036210480 -> 140193036713840
140193036210480 [label=ToCopyBackward0]
140193036210864 -> 140193036210480
140193039470480 [label="encoder.layer.6.experts.experts.0.dense2.bias
(768)" fillcolor=lightblue]
140193039470480 -> 140193036210864
140193036210864 [label=AccumulateGrad]
140193036210288 -> 140193036713840
140193036210288 [label=ViewBackward0]
140193036210768 -> 140193036210288
140193036210768 [label=GeluBackward0]
140193036210816 -> 140193036210768
140193036210816 [label=ViewBackward0]
140193036211056 -> 140193036210816
140193036211056 [label=AddmmBackward0]
140193036211248 -> 140193036211056
140193036211248 [label=ToCopyBackward0]
140193036211536 -> 140193036211248
140193039470400 [label="encoder.layer.6.experts.experts.0.dense1.bias
(3072)" fillcolor=lightblue]
140193039470400 -> 140193036211536
140193036211536 [label=AccumulateGrad]
140193036211344 -> 140193036211056
140193036211344 [label=ViewBackward0]
140193036211824 -> 140193036211344
140193036211824 [label=ToCopyBackward0]
140193036211920 -> 140193036211824
140193036211920 [label=SliceBackward0]
140193036212112 -> 140193036211920
140193036212112 [label=SliceBackward0]
140193036212304 -> 140193036212112
140193036212304 [label=NativeLayerNormBackward0]
140193036212400 -> 140193036212304
140193036212400 [label=AddBackward0]
140193036212784 -> 140193036212400
140193036212784 [label=NativeDropoutBackward0]
140193036212736 -> 140193036212784
140193036212736 [label=ViewBackward0]
140193036212976 -> 140193036212736
140193036212976 [label=AddmmBackward0]
140193036213168 -> 140193036212976
140193036213168 [label=ToCopyBackward0]
140193036213456 -> 140193036213168
140193039490320 [label="encoder.layer.6.crossattention.output.dense.bias
(768)" fillcolor=lightblue]
140193039490320 -> 140193036213456
140193036213456 [label=AccumulateGrad]
140193036213264 -> 140193036212976
140193036213264 [label=ViewBackward0]
140193036213744 -> 140193036213264
140193036213744 [label=ViewBackward0]
140193036213840 -> 140193036213744
140193036213840 [label=CloneBackward0]
140193036214032 -> 140193036213840
140193036214032 [label=PermuteBackward0]
140193036214224 -> 140193036214032
140193036214224 [label=UnsafeViewBackward0]
140193036214128 -> 140193036214224
140193036214128 [label=BmmBackward0]
140193036247344 -> 140193036214128
140193036247344 [label=ReshapeAliasBackward0]
140193036247440 -> 140193036247344
140193036247440 [label=ExpandBackward0]
140193036247488 -> 140193036247440
140193036247488 [label=ToCopyBackward0]
140193036247728 -> 140193036247488
140193036247728 [label=NativeDropoutBackward0]
140193036247920 -> 140193036247728
140193036247920 [label=SoftmaxBackward0]
140193036247968 -> 140193036247920
140193036247968 [label=AddBackward0]
140193036248208 -> 140193036247968
140193036248208 [label=DivBackward0]
140193036248400 -> 140193036248208
140193036248400 [label=UnsafeViewBackward0]
140193036248448 -> 140193036248400
140193036248448 [label=BmmBackward0]
140193036248688 -> 140193036248448
140193036248688 [label=UnsafeViewBackward0]
140193036249072 -> 140193036248688
140193036249072 [label=CloneBackward0]
140193036249264 -> 140193036249072
140193036249264 [label=ExpandBackward0]
140193036249456 -> 140193036249264
140193036249456 [label=PermuteBackward0]
140193036249552 -> 140193036249456
140193036249552 [label=ViewBackward0]
140193036249744 -> 140193036249552
140193036249744 [label=ViewBackward0]
140193036249936 -> 140193036249744
140193036249936 [label=AddmmBackward0]
140193036250032 -> 140193036249936
140193036250032 [label=ToCopyBackward0]
140193036250416 -> 140193036250032
140193039490960 [label="encoder.layer.6.crossattention.self.query.bias
(768)" fillcolor=lightblue]
140193039490960 -> 140193036250416
140193036250416 [label=AccumulateGrad]
140193036249840 -> 140193036249936
140193036249840 [label=ViewBackward0]
140193036250320 -> 140193036249840
140193036250320 [label=ToCopyBackward0]
140193036212496 -> 140193036250320
140193036212496 [label=SliceBackward0]
140193036250704 -> 140193036212496
140193036250704 [label=SliceBackward0]
140193036250896 -> 140193036250704
140193036250896 [label=SliceBackward0]
140193036250992 -> 140193036250896
140193036250992 [label=NativeLayerNormBackward0]
140193036250848 -> 140193036250992
140193036250848 [label=AddBackward0]
140193036272016 -> 140193036250848
140193036272016 [label=NativeDropoutBackward0]
140193036272112 -> 140193036272016
140193036272112 [label=ViewBackward0]
140193036272304 -> 140193036272112
140193036272304 [label=AddmmBackward0]
140193036272352 -> 140193036272304
140193036272352 [label=ToCopyBackward0]
140193036272784 -> 140193036272352
140193039495712 [label="encoder.layer.6.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039495712 -> 140193036272784
140193036272784 [label=AccumulateGrad]
140193036272496 -> 140193036272304
140193036272496 [label=ViewBackward0]
140193036272976 -> 140193036272496
140193036272976 [label=ViewBackward0]
140193036273168 -> 140193036272976
140193036273168 [label=CloneBackward0]
140193036273360 -> 140193036273168
140193036273360 [label=PermuteBackward0]
140193036273456 -> 140193036273360
140193036273456 [label=UnsafeViewBackward0]
140193036273648 -> 140193036273456
140193036273648 [label=BmmBackward0]
140193036273840 -> 140193036273648
140193036273840 [label=ReshapeAliasBackward0]
140193036273792 -> 140193036273840
140193036273792 [label=ExpandBackward0]
140193036274032 -> 140193036273792
140193036274032 [label=ToCopyBackward0]
140193036274224 -> 140193036274032
140193036274224 [label=NativeDropoutBackward0]
140193036274608 -> 140193036274224
140193036274608 [label=SoftmaxBackward0]
140193036274800 -> 140193036274608
140193036274800 [label=AddBackward0]
140193036274896 -> 140193036274800
140193036274896 [label=DivBackward0]
140193036275088 -> 140193036274896
140193036275088 [label=UnsafeViewBackward0]
140193036275280 -> 140193036275088
140193036275280 [label=BmmBackward0]
140193036275376 -> 140193036275280
140193036275376 [label=UnsafeViewBackward0]
140193036275472 -> 140193036275376
140193036275472 [label=CloneBackward0]
140193036275232 -> 140193036275472
140193036275232 [label=ExpandBackward0]
140193036302992 -> 140193036275232
140193036302992 [label=PermuteBackward0]
140193036303760 -> 140193036302992
140193036303760 [label=ViewBackward0]
140193036303712 -> 140193036303760
140193036303712 [label=ViewBackward0]
140193036300400 -> 140193036303712
140193036300400 [label=AddmmBackward0]
140193036300496 -> 140193036300400
140193036300496 [label=ToCopyBackward0]
140193036300784 -> 140193036300496
140193039496432 [label="encoder.layer.6.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039496432 -> 140193036300784
140193036300784 [label=AccumulateGrad]
140193036300448 -> 140193036300400
140193036300448 [label=ViewBackward0]
140193036300976 -> 140193036300448
140193036300976 [label=ToCopyBackward0]
140193036271824 -> 140193036300976
140193036271824 [label=CatBackward0]
140193036301072 -> 140193036271824
140193036301072 [label=NativeLayerNormBackward0]
140193036301456 -> 140193036301072
140193036301456 [label=AddBackward0]
140193036301648 -> 140193036301456
140193036301648 [label=NativeDropoutBackward0]
140193036301792 -> 140193036301648
140193036301792 [label=ViewBackward0]
140193036302032 -> 140193036301792
140193036302032 [label=AddmmBackward0]
140193036302224 -> 140193036302032
140193036302224 [label=ToCopyBackward0]
140193036303568 -> 140193036302224
140193039496912 [label="encoder.layer.5.experts.dense2.bias
(768)" fillcolor=lightblue]
140193039496912 -> 140193036303568
140193036303568 [label=AccumulateGrad]
140193036302128 -> 140193036302032
140193036302128 [label=ViewBackward0]
140193036302608 -> 140193036302128
140193036302608 [label=GeluBackward0]
140193036303280 -> 140193036302608
140193036303280 [label=ViewBackward0]
140193036302896 -> 140193036303280
140193036302896 [label=AddmmBackward0]
140193036304240 -> 140193036302896
140193036304240 [label=ToCopyBackward0]
140193036302272 -> 140193036304240
140193039497152 [label="encoder.layer.5.experts.dense1.bias
(3072)" fillcolor=lightblue]
140193039497152 -> 140193036302272
140193036302272 [label=AccumulateGrad]
140193036303664 -> 140193036302896
140193036303664 [label=ViewBackward0]
140193036303184 -> 140193036303664
140193036303184 [label=ToCopyBackward0]
140193036301552 -> 140193036303184
140193036301552 [label=SliceBackward0]
140193037273120 -> 140193036301552
140193037273120 [label=SliceBackward0]
140193037273024 -> 140193037273120
140193037273024 [label=SliceBackward0]
140193037272928 -> 140193037273024
140193037272928 [label=SliceBackward0]
140193037272832 -> 140193037272928
140193037272832 [label=SliceBackward0]
140193037272736 -> 140193037272832
140193037272736 [label=NativeLayerNormBackward0]
140193037272640 -> 140193037272736
140193037272640 [label=AddBackward0]
140193037272448 -> 140193037272640
140193037272448 [label=NativeDropoutBackward0]
140193037272400 -> 140193037272448
140193037272400 [label=ViewBackward0]
140193037272304 -> 140193037272400
140193037272304 [label=AddmmBackward0]
140193037272208 -> 140193037272304
140193037272208 [label=ToCopyBackward0]
140193037272016 -> 140193037272208
140193039499072 [label="encoder.layer.5.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039499072 -> 140193037272016
140193037272016 [label=AccumulateGrad]
140193037272160 -> 140193037272304
140193037272160 [label=ViewBackward0]
140193037271872 -> 140193037272160
140193037271872 [label=ViewBackward0]
140193037271776 -> 140193037271872
140193037271776 [label=CloneBackward0]
140193037271680 -> 140193037271776
140193037271680 [label=PermuteBackward0]
140193037271584 -> 140193037271680
140193037271584 [label=UnsafeViewBackward0]
140193037271488 -> 140193037271584
140193037271488 [label=BmmBackward0]
140193037271392 -> 140193037271488
140193037271392 [label=ReshapeAliasBackward0]
140193037271344 -> 140193037271392
140193037271344 [label=ExpandBackward0]
140193037271248 -> 140193037271344
140193037271248 [label=ToCopyBackward0]
140193037271152 -> 140193037271248
140193037271152 [label=NativeDropoutBackward0]
140193037273408 -> 140193037271152
140193037273408 [label=SoftmaxBackward0]
140193037273504 -> 140193037273408
140193037273504 [label=AddBackward0]
140193037273600 -> 140193037273504
140193037273600 [label=DivBackward0]
140193037273696 -> 140193037273600
140193037273696 [label=UnsafeViewBackward0]
140193037273792 -> 140193037273696
140193037273792 [label=BmmBackward0]
140193037273888 -> 140193037273792
140193037273888 [label=UnsafeViewBackward0]
140193037274032 -> 140193037273888
140193037274032 [label=CloneBackward0]
140193037274128 -> 140193037274032
140193037274128 [label=ExpandBackward0]
140193037274224 -> 140193037274128
140193037274224 [label=PermuteBackward0]
140193037274320 -> 140193037274224
140193037274320 [label=ViewBackward0]
140193037274416 -> 140193037274320
140193037274416 [label=ViewBackward0]
140193037274512 -> 140193037274416
140193037274512 [label=AddmmBackward0]
140193037274608 -> 140193037274512
140193037274608 [label=ToCopyBackward0]
140193037274800 -> 140193037274608
140193039516272 [label="encoder.layer.5.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039516272 -> 140193037274800
140193037274800 [label=AccumulateGrad]
140193037274560 -> 140193037274512
140193037274560 [label=ViewBackward0]
140193037274848 -> 140193037274560
140193037274848 [label=ToCopyBackward0]
140193037272592 -> 140193037274848
140193037272592 [label=CatBackward0]
140193037274992 -> 140193037272592
140193037274992 [label=NativeLayerNormBackward0]
140193037275088 -> 140193037274992
140193037275088 [label=AddBackward0]
140193037361408 -> 140193037275088
140193037361408 [label=NativeDropoutBackward0]
140193037361552 -> 140193037361408
140193037361552 [label=ViewBackward0]
140193037361648 -> 140193037361552
140193037361648 [label=AddmmBackward0]
140193037361744 -> 140193037361648
140193037361744 [label=ToCopyBackward0]
140193037361936 -> 140193037361744
140193039516752 [label="encoder.layer.4.experts.dense2.bias
(768)" fillcolor=lightblue]
140193039516752 -> 140193037361936
140193037361936 [label=AccumulateGrad]
140193037361696 -> 140193037361648
140193037361696 [label=ViewBackward0]
140193037361984 -> 140193037361696
140193037361984 [label=GeluBackward0]
140193037362080 -> 140193037361984
140193037362080 [label=ViewBackward0]
140193037362176 -> 140193037362080
140193037362176 [label=AddmmBackward0]
140193037362272 -> 140193037362176
140193037362272 [label=ToCopyBackward0]
140193037362464 -> 140193037362272
140193039516992 [label="encoder.layer.4.experts.dense1.bias
(3072)" fillcolor=lightblue]
140193039516992 -> 140193037362464
140193037362464 [label=AccumulateGrad]
140193037362224 -> 140193037362176
140193037362224 [label=ViewBackward0]
140193037362512 -> 140193037362224
140193037362512 [label=ToCopyBackward0]
140193037361360 -> 140193037362512
140193037361360 [label=SliceBackward0]
140193037362656 -> 140193037361360
140193037362656 [label=SliceBackward0]
140193037362752 -> 140193037362656
140193037362752 [label=NativeLayerNormBackward0]
140193037362848 -> 140193037362752
140193037362848 [label=AddBackward0]
140193037363040 -> 140193037362848
140193037363040 [label=NativeDropoutBackward0]
140193037363184 -> 140193037363040
140193037363184 [label=ViewBackward0]
140193037363280 -> 140193037363184
140193037363280 [label=AddmmBackward0]
140193037363376 -> 140193037363280
140193037363376 [label=ToCopyBackward0]
140193037363568 -> 140193037363376
140193039518592 [label="encoder.layer.4.crossattention.output.dense.bias
(768)" fillcolor=lightblue]
140193039518592 -> 140193037363568
140193037363568 [label=AccumulateGrad]
140193037363328 -> 140193037363280
140193037363328 [label=ViewBackward0]
140193037363616 -> 140193037363328
140193037363616 [label=ViewBackward0]
140193037363712 -> 140193037363616
140193037363712 [label=CloneBackward0]
140193037363808 -> 140193037363712
140193037363808 [label=PermuteBackward0]
140193037363904 -> 140193037363808
140193037363904 [label=UnsafeViewBackward0]
140193037364000 -> 140193037363904
140193037364000 [label=BmmBackward0]
140193037364096 -> 140193037364000
140193037364096 [label=ReshapeAliasBackward0]
140193037364240 -> 140193037364096
140193037364240 [label=ExpandBackward0]
140193037364336 -> 140193037364240
140193037364336 [label=ToCopyBackward0]
140193037364432 -> 140193037364336
140193037364432 [label=NativeDropoutBackward0]
140193037364528 -> 140193037364432
140193037364528 [label=SoftmaxBackward0]
140193037364624 -> 140193037364528
140193037364624 [label=AddBackward0]
140193037364720 -> 140193037364624
140193037364720 [label=DivBackward0]
140193037364816 -> 140193037364720
140193037364816 [label=UnsafeViewBackward0]
140193037364912 -> 140193037364816
140193037364912 [label=BmmBackward0]
140193037365008 -> 140193037364912
140193037365008 [label=UnsafeViewBackward0]
140193037365152 -> 140193037365008
140193037365152 [label=CloneBackward0]
140193037365200 -> 140193037365152
140193037365200 [label=ExpandBackward0]
140193037275296 -> 140193037365200
140193037275296 [label=PermuteBackward0]
140193037275392 -> 140193037275296
140193037275392 [label=ViewBackward0]
140193037275488 -> 140193037275392
140193037275488 [label=ViewBackward0]
140193037275584 -> 140193037275488
140193037275584 [label=AddmmBackward0]
140193037275680 -> 140193037275584
140193037275680 [label=ToCopyBackward0]
140193037275872 -> 140193037275680
140193039519312 [label="encoder.layer.4.crossattention.self.query.bias
(768)" fillcolor=lightblue]
140193039519312 -> 140193037275872
140193037275872 [label=AccumulateGrad]
140193037275632 -> 140193037275584
140193037275632 [label=ViewBackward0]
140193037275920 -> 140193037275632
140193037275920 [label=ToCopyBackward0]
140193037362992 -> 140193037275920
140193037362992 [label=SliceBackward0]
140193037276064 -> 140193037362992
140193037276064 [label=SliceBackward0]
140193037276160 -> 140193037276064
140193037276160 [label=SliceBackward0]
140193037276256 -> 140193037276160
140193037276256 [label=NativeLayerNormBackward0]
140193037276352 -> 140193037276256
140193037276352 [label=AddBackward0]
140193037276544 -> 140193037276352
140193037276544 [label=NativeDropoutBackward0]
140193037276688 -> 140193037276544
140193037276688 [label=ViewBackward0]
140193037276784 -> 140193037276688
140193037276784 [label=AddmmBackward0]
140193037276880 -> 140193037276784
140193037276880 [label=ToCopyBackward0]
140193037277072 -> 140193037276880
140193039536272 [label="encoder.layer.4.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039536272 -> 140193037277072
140193037277072 [label=AccumulateGrad]
140193037276832 -> 140193037276784
140193037276832 [label=ViewBackward0]
140193037277120 -> 140193037276832
140193037277120 [label=ViewBackward0]
140193037277216 -> 140193037277120
140193037277216 [label=CloneBackward0]
140193037277312 -> 140193037277216
140193037277312 [label=PermuteBackward0]
140193037277408 -> 140193037277312
140193037277408 [label=UnsafeViewBackward0]
140193037277504 -> 140193037277408
140193037277504 [label=BmmBackward0]
140193037277600 -> 140193037277504
140193037277600 [label=ReshapeAliasBackward0]
140193037277744 -> 140193037277600
140193037277744 [label=ExpandBackward0]
140193037277840 -> 140193037277744
140193037277840 [label=ToCopyBackward0]
140193037277936 -> 140193037277840
140193037277936 [label=NativeDropoutBackward0]
140193037278032 -> 140193037277936
140193037278032 [label=SoftmaxBackward0]
140193037278128 -> 140193037278032
140193037278128 [label=AddBackward0]
140193037278224 -> 140193037278128
140193037278224 [label=DivBackward0]
140193037278320 -> 140193037278224
140193037278320 [label=UnsafeViewBackward0]
140193037278416 -> 140193037278320
140193037278416 [label=BmmBackward0]
140193037278512 -> 140193037278416
140193037278512 [label=UnsafeViewBackward0]
140193037278656 -> 140193037278512
140193037278656 [label=CloneBackward0]
140193037278752 -> 140193037278656
140193037278752 [label=ExpandBackward0]
140193037278848 -> 140193037278752
140193037278848 [label=PermuteBackward0]
140193037278944 -> 140193037278848
140193037278944 [label=ViewBackward0]
140193037279040 -> 140193037278944
140193037279040 [label=ViewBackward0]
140193037279136 -> 140193037279040
140193037279136 [label=AddmmBackward0]
140193037279184 -> 140193037279136
140193037279184 [label=ToCopyBackward0]
140193037259008 -> 140193037279184
140193039536992 [label="encoder.layer.4.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039536992 -> 140193037259008
140193037259008 [label=AccumulateGrad]
140193037278560 -> 140193037279136
140193037278560 [label=ViewBackward0]
140193037259056 -> 140193037278560
140193037259056 [label=ToCopyBackward0]
140193037276496 -> 140193037259056
140193037276496 [label=CatBackward0]
140193037259200 -> 140193037276496
140193037259200 [label=NativeLayerNormBackward0]
140193037259344 -> 140193037259200
140193037259344 [label=AddBackward0]
140193037259536 -> 140193037259344
140193037259536 [label=NativeDropoutBackward0]
140193037259680 -> 140193037259536
140193037259680 [label=ViewBackward0]
140193037259776 -> 140193037259680
140193037259776 [label=AddmmBackward0]
140193037259872 -> 140193037259776
140193037259872 [label=ToCopyBackward0]
140193037260064 -> 140193037259872
140193039537472 [label="encoder.layer.3.experts.dense2.bias
(768)" fillcolor=lightblue]
140193039537472 -> 140193037260064
140193037260064 [label=AccumulateGrad]
140193037259824 -> 140193037259776
140193037259824 [label=ViewBackward0]
140193037260112 -> 140193037259824
140193037260112 [label=GeluBackward0]
140193037260208 -> 140193037260112
140193037260208 [label=ViewBackward0]
140193037260304 -> 140193037260208
140193037260304 [label=AddmmBackward0]
140193037260400 -> 140193037260304
140193037260400 [label=ToCopyBackward0]
140193037260592 -> 140193037260400
140193039537712 [label="encoder.layer.3.experts.dense1.bias
(3072)" fillcolor=lightblue]
140193039537712 -> 140193037260592
140193037260592 [label=AccumulateGrad]
140193037260352 -> 140193037260304
140193037260352 [label=ViewBackward0]
140193037260640 -> 140193037260352
140193037260640 [label=ToCopyBackward0]
140193037259488 -> 140193037260640
140193037259488 [label=SliceBackward0]
140193037260784 -> 140193037259488
140193037260784 [label=SliceBackward0]
140193037260880 -> 140193037260784
140193037260880 [label=SliceBackward0]
140193037260976 -> 140193037260880
140193037260976 [label=SliceBackward0]
140193037261072 -> 140193037260976
140193037261072 [label=SliceBackward0]
140193037261168 -> 140193037261072
140193037261168 [label=NativeLayerNormBackward0]
140193037261264 -> 140193037261168
140193037261264 [label=AddBackward0]
140193037261456 -> 140193037261264
140193037261456 [label=NativeDropoutBackward0]
140193037261600 -> 140193037261456
140193037261600 [label=ViewBackward0]
140193037261696 -> 140193037261600
140193037261696 [label=AddmmBackward0]
140193037261792 -> 140193037261696
140193037261792 [label=ToCopyBackward0]
140193037261984 -> 140193037261792
140193039539632 [label="encoder.layer.3.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039539632 -> 140193037261984
140193037261984 [label=AccumulateGrad]
140193037261744 -> 140193037261696
140193037261744 [label=ViewBackward0]
140193037262032 -> 140193037261744
140193037262032 [label=ViewBackward0]
140193037262128 -> 140193037262032
140193037262128 [label=CloneBackward0]
140193037262224 -> 140193037262128
140193037262224 [label=PermuteBackward0]
140193037262320 -> 140193037262224
140193037262320 [label=UnsafeViewBackward0]
140193037262416 -> 140193037262320
140193037262416 [label=BmmBackward0]
140193037262512 -> 140193037262416
140193037262512 [label=ReshapeAliasBackward0]
140193037262656 -> 140193037262512
140193037262656 [label=ExpandBackward0]
140193037262752 -> 140193037262656
140193037262752 [label=ToCopyBackward0]
140193037262800 -> 140193037262752
140193037262800 [label=NativeDropoutBackward0]
140193035952288 -> 140193037262800
140193035952288 [label=SoftmaxBackward0]
140193035952384 -> 140193035952288
140193035952384 [label=AddBackward0]
140193035952480 -> 140193035952384
140193035952480 [label=DivBackward0]
140193035952576 -> 140193035952480
140193035952576 [label=UnsafeViewBackward0]
140193035952672 -> 140193035952576
140193035952672 [label=BmmBackward0]
140193035952768 -> 140193035952672
140193035952768 [label=UnsafeViewBackward0]
140193035952912 -> 140193035952768
140193035952912 [label=CloneBackward0]
140193035953008 -> 140193035952912
140193035953008 [label=ExpandBackward0]
140193035953104 -> 140193035953008
140193035953104 [label=PermuteBackward0]
140193035953200 -> 140193035953104
140193035953200 [label=ViewBackward0]
140193035953296 -> 140193035953200
140193035953296 [label=ViewBackward0]
140193035953392 -> 140193035953296
140193035953392 [label=AddmmBackward0]
140193035953488 -> 140193035953392
140193035953488 [label=ToCopyBackward0]
140193035953680 -> 140193035953488
140193039548640 [label="encoder.layer.3.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039548640 -> 140193035953680
140193035953680 [label=AccumulateGrad]
140193035953440 -> 140193035953392
140193035953440 [label=ViewBackward0]
140193035953728 -> 140193035953440
140193035953728 [label=ToCopyBackward0]
140193037261408 -> 140193035953728
140193037261408 [label=CatBackward0]
140193035953872 -> 140193037261408
140193035953872 [label=NativeLayerNormBackward0]
140193035954016 -> 140193035953872
140193035954016 [label=AddBackward0]
140193035954208 -> 140193035954016
140193035954208 [label=NativeDropoutBackward0]
140193035954352 -> 140193035954208
140193035954352 [label=ViewBackward0]
140193035954448 -> 140193035954352
140193035954448 [label=AddmmBackward0]
140193035954544 -> 140193035954448
140193035954544 [label=ToCopyBackward0]
140193035954736 -> 140193035954544
140193039549120 [label="encoder.layer.2.experts.dense2.bias
(768)" fillcolor=lightblue]
140193039549120 -> 140193035954736
140193035954736 [label=AccumulateGrad]
140193035954496 -> 140193035954448
140193035954496 [label=ViewBackward0]
140193035954784 -> 140193035954496
140193035954784 [label=GeluBackward0]
140193035954880 -> 140193035954784
140193035954880 [label=ViewBackward0]
140193035954976 -> 140193035954880
140193035954976 [label=AddmmBackward0]
140193035955072 -> 140193035954976
140193035955072 [label=ToCopyBackward0]
140193035955264 -> 140193035955072
140193039549360 [label="encoder.layer.2.experts.dense1.bias
(3072)" fillcolor=lightblue]
140193039549360 -> 140193035955264
140193035955264 [label=AccumulateGrad]
140193035955024 -> 140193035954976
140193035955024 [label=ViewBackward0]
140193035955312 -> 140193035955024
140193035955312 [label=ToCopyBackward0]
140193035954160 -> 140193035955312
140193035954160 [label=SliceBackward0]
140193035955456 -> 140193035954160
140193035955456 [label=SliceBackward0]
140193035955552 -> 140193035955456
140193035955552 [label=NativeLayerNormBackward0]
140193035955648 -> 140193035955552
140193035955648 [label=AddBackward0]
140193035955840 -> 140193035955648
140193035955840 [label=NativeDropoutBackward0]
140193035955984 -> 140193035955840
140193035955984 [label=ViewBackward0]
140193035956080 -> 140193035955984
140193035956080 [label=AddmmBackward0]
140193035956176 -> 140193035956080
140193035956176 [label=ToCopyBackward0]
140193035960528 -> 140193035956176
140193039551280 [label="encoder.layer.2.crossattention.output.dense.bias
(768)" fillcolor=lightblue]
140193039551280 -> 140193035960528
140193035960528 [label=AccumulateGrad]
140193035956128 -> 140193035956080
140193035956128 [label=ViewBackward0]
140193035960576 -> 140193035956128
140193035960576 [label=ViewBackward0]
140193035960672 -> 140193035960576
140193035960672 [label=CloneBackward0]
140193035960768 -> 140193035960672
140193035960768 [label=PermuteBackward0]
140193035960864 -> 140193035960768
140193035960864 [label=UnsafeViewBackward0]
140193035960960 -> 140193035960864
140193035960960 [label=BmmBackward0]
140193035961056 -> 140193035960960
140193035961056 [label=ReshapeAliasBackward0]
140193035961200 -> 140193035961056
140193035961200 [label=ExpandBackward0]
140193035961296 -> 140193035961200
140193035961296 [label=ToCopyBackward0]
140193035961392 -> 140193035961296
140193035961392 [label=NativeDropoutBackward0]
140193035961488 -> 140193035961392
140193035961488 [label=SoftmaxBackward0]
140193035961584 -> 140193035961488
140193035961584 [label=AddBackward0]
140193035961680 -> 140193035961584
140193035961680 [label=DivBackward0]
140193035961776 -> 140193035961680
140193035961776 [label=UnsafeViewBackward0]
140193035961872 -> 140193035961776
140193035961872 [label=BmmBackward0]
140193035961968 -> 140193035961872
140193035961968 [label=UnsafeViewBackward0]
140193035962112 -> 140193035961968
140193035962112 [label=CloneBackward0]
140193035962208 -> 140193035962112
140193035962208 [label=ExpandBackward0]
140193035962304 -> 140193035962208
140193035962304 [label=PermuteBackward0]
140193035962400 -> 140193035962304
140193035962400 [label=ViewBackward0]
140193035962496 -> 140193035962400
140193035962496 [label=ViewBackward0]
140193035962592 -> 140193035962496
140193035962592 [label=AddmmBackward0]
140193035962688 -> 140193035962592
140193035962688 [label=ToCopyBackward0]
140193035962880 -> 140193035962688
140193039552000 [label="encoder.layer.2.crossattention.self.query.bias
(768)" fillcolor=lightblue]
140193039552000 -> 140193035962880
140193035962880 [label=AccumulateGrad]
140193035962640 -> 140193035962592
140193035962640 [label=ViewBackward0]
140193035962928 -> 140193035962640
140193035962928 [label=ToCopyBackward0]
140193035955792 -> 140193035962928
140193035955792 [label=SliceBackward0]
140193035963072 -> 140193035955792
140193035963072 [label=SliceBackward0]
140193035963168 -> 140193035963072
140193035963168 [label=SliceBackward0]
140193035963264 -> 140193035963168
140193035963264 [label=NativeLayerNormBackward0]
140193035963360 -> 140193035963264
140193035963360 [label=AddBackward0]
140193035963552 -> 140193035963360
140193035963552 [label=NativeDropoutBackward0]
140193035963696 -> 140193035963552
140193035963696 [label=ViewBackward0]
140193035963792 -> 140193035963696
140193035963792 [label=AddmmBackward0]
140193035963888 -> 140193035963792
140193035963888 [label=ToCopyBackward0]
140193035964080 -> 140193035963888
140193039556672 [label="encoder.layer.2.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039556672 -> 140193035964080
140193035964080 [label=AccumulateGrad]
140193035963840 -> 140193035963792
140193035963840 [label=ViewBackward0]
140193035964128 -> 140193035963840
140193035964128 [label=ViewBackward0]
140193035964224 -> 140193035964128
140193035964224 [label=CloneBackward0]
140193035964320 -> 140193035964224
140193035964320 [label=PermuteBackward0]
140193035964368 -> 140193035964320
140193035964368 [label=UnsafeViewBackward0]
140193035985056 -> 140193035964368
140193035985056 [label=BmmBackward0]
140193035985152 -> 140193035985056
140193035985152 [label=ReshapeAliasBackward0]
140193035985296 -> 140193035985152
140193035985296 [label=ExpandBackward0]
140193035985392 -> 140193035985296
140193035985392 [label=ToCopyBackward0]
140193035985488 -> 140193035985392
140193035985488 [label=NativeDropoutBackward0]
140193035985584 -> 140193035985488
140193035985584 [label=SoftmaxBackward0]
140193035985680 -> 140193035985584
140193035985680 [label=AddBackward0]
140193035985776 -> 140193035985680
140193035985776 [label=DivBackward0]
140193035985872 -> 140193035985776
140193035985872 [label=UnsafeViewBackward0]
140193035985968 -> 140193035985872
140193035985968 [label=BmmBackward0]
140193035986064 -> 140193035985968
140193035986064 [label=UnsafeViewBackward0]
140193035986208 -> 140193035986064
140193035986208 [label=CloneBackward0]
140193035986304 -> 140193035986208
140193035986304 [label=ExpandBackward0]
140193035986400 -> 140193035986304
140193035986400 [label=PermuteBackward0]
140193035986496 -> 140193035986400
140193035986496 [label=ViewBackward0]
140193035986592 -> 140193035986496
140193035986592 [label=ViewBackward0]
140193035986688 -> 140193035986592
140193035986688 [label=AddmmBackward0]
140193035986784 -> 140193035986688
140193035986784 [label=ToCopyBackward0]
140193035986976 -> 140193035986784
140193039557392 [label="encoder.layer.2.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039557392 -> 140193035986976
140193035986976 [label=AccumulateGrad]
140193035986736 -> 140193035986688
140193035986736 [label=ViewBackward0]
140193035987024 -> 140193035986736
140193035987024 [label=ToCopyBackward0]
140193035963504 -> 140193035987024
140193035963504 [label=CatBackward0]
140193035987168 -> 140193035963504
140193035987168 [label=NativeLayerNormBackward0]
140193035987312 -> 140193035987168
140193035987312 [label=AddBackward0]
140193035987504 -> 140193035987312
140193035987504 [label=NativeDropoutBackward0]
140193035987648 -> 140193035987504
140193035987648 [label=ViewBackward0]
140193035987744 -> 140193035987648
140193035987744 [label=AddmmBackward0]
140193035987840 -> 140193035987744
140193035987840 [label=ToCopyBackward0]
140193035988032 -> 140193035987840
140193039557872 [label="encoder.layer.1.experts.dense2.bias
(768)" fillcolor=lightblue]
140193039557872 -> 140193035988032
140193035988032 [label=AccumulateGrad]
140193035987792 -> 140193035987744
140193035987792 [label=ViewBackward0]
140193035988080 -> 140193035987792
140193035988080 [label=GeluBackward0]
140193035988176 -> 140193035988080
140193035988176 [label=ViewBackward0]
140193035988272 -> 140193035988176
140193035988272 [label=AddmmBackward0]
140193035988368 -> 140193035988272
140193035988368 [label=ToCopyBackward0]
140193035988560 -> 140193035988368
140193039558112 [label="encoder.layer.1.experts.dense1.bias
(3072)" fillcolor=lightblue]
140193039558112 -> 140193035988560
140193035988560 [label=AccumulateGrad]
140193035988320 -> 140193035988272
140193035988320 [label=ViewBackward0]
140193035988608 -> 140193035988320
140193035988608 [label=ToCopyBackward0]
140193035987456 -> 140193035988608
140193035987456 [label=SliceBackward0]
140193035988752 -> 140193035987456
140193035988752 [label=SliceBackward0]
140193035988848 -> 140193035988752
140193035988848 [label=SliceBackward0]
140193035988944 -> 140193035988848
140193035988944 [label=SliceBackward0]
140193035988464 -> 140193035988944
140193035988464 [label=SliceBackward0]
140193036001488 -> 140193035988464
140193036001488 [label=NativeLayerNormBackward0]
140193036001584 -> 140193036001488
140193036001584 [label=AddBackward0]
140193036001776 -> 140193036001584
140193036001776 [label=NativeDropoutBackward0]
140193036001920 -> 140193036001776
140193036001920 [label=ViewBackward0]
140193036002016 -> 140193036001920
140193036002016 [label=AddmmBackward0]
140193036002112 -> 140193036002016
140193036002112 [label=ToCopyBackward0]
140193036002304 -> 140193036002112
140193039560032 [label="encoder.layer.1.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039560032 -> 140193036002304
140193036002304 [label=AccumulateGrad]
140193036002064 -> 140193036002016
140193036002064 [label=ViewBackward0]
140193036002352 -> 140193036002064
140193036002352 [label=ViewBackward0]
140193036002448 -> 140193036002352
140193036002448 [label=CloneBackward0]
140193036002544 -> 140193036002448
140193036002544 [label=PermuteBackward0]
140193036002640 -> 140193036002544
140193036002640 [label=UnsafeViewBackward0]
140193036002736 -> 140193036002640
140193036002736 [label=BmmBackward0]
140193036002832 -> 140193036002736
140193036002832 [label=ReshapeAliasBackward0]
140193036002976 -> 140193036002832
140193036002976 [label=ExpandBackward0]
140193036003072 -> 140193036002976
140193036003072 [label=ToCopyBackward0]
140193036003168 -> 140193036003072
140193036003168 [label=NativeDropoutBackward0]
140193036003264 -> 140193036003168
140193036003264 [label=SoftmaxBackward0]
140193036003360 -> 140193036003264
140193036003360 [label=AddBackward0]
140193036003456 -> 140193036003360
140193036003456 [label=DivBackward0]
140193036003552 -> 140193036003456
140193036003552 [label=UnsafeViewBackward0]
140193036003648 -> 140193036003552
140193036003648 [label=BmmBackward0]
140193036003744 -> 140193036003648
140193036003744 [label=UnsafeViewBackward0]
140193036003888 -> 140193036003744
140193036003888 [label=CloneBackward0]
140193036003984 -> 140193036003888
140193036003984 [label=ExpandBackward0]
140193036004080 -> 140193036003984
140193036004080 [label=PermuteBackward0]
140193036004176 -> 140193036004080
140193036004176 [label=ViewBackward0]
140193036004272 -> 140193036004176
140193036004272 [label=ViewBackward0]
140193036004368 -> 140193036004272
140193036004368 [label=AddmmBackward0]
140193036004464 -> 140193036004368
140193036004464 [label=ToCopyBackward0]
140193036004656 -> 140193036004464
140193039577232 [label="encoder.layer.1.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039577232 -> 140193036004656
140193036004656 [label=AccumulateGrad]
140193036004416 -> 140193036004368
140193036004416 [label=ViewBackward0]
140193036004704 -> 140193036004416
140193036004704 [label=ToCopyBackward0]
140193036001728 -> 140193036004704
140193036001728 [label=CatBackward0]
140193036004848 -> 140193036001728
140193036004848 [label=NativeLayerNormBackward0]
140193036004992 -> 140193036004848
140193036004992 [label=AddBackward0]
140193036005184 -> 140193036004992
140193036005184 [label=NativeDropoutBackward0]
140193036005328 -> 140193036005184
140193036005328 [label=ViewBackward0]
140193036005232 -> 140193036005328
140193036005232 [label=AddmmBackward0]
140193036017872 -> 140193036005232
140193036017872 [label=ToCopyBackward0]
140193036018064 -> 140193036017872
140193039577712 [label="encoder.layer.0.experts.dense2.bias
(768)" fillcolor=lightblue]
140193039577712 -> 140193036018064
140193036018064 [label=AccumulateGrad]
140193036017824 -> 140193036005232
140193036017824 [label=ViewBackward0]
140193036018112 -> 140193036017824
140193036018112 [label=GeluBackward0]
140193036018208 -> 140193036018112
140193036018208 [label=ViewBackward0]
140193036018304 -> 140193036018208
140193036018304 [label=AddmmBackward0]
140193036018400 -> 140193036018304
140193036018400 [label=ToCopyBackward0]
140193036018592 -> 140193036018400
140193039577952 [label="encoder.layer.0.experts.dense1.bias
(3072)" fillcolor=lightblue]
140193039577952 -> 140193036018592
140193036018592 [label=AccumulateGrad]
140193036018352 -> 140193036018304
140193036018352 [label=ViewBackward0]
140193036018640 -> 140193036018352
140193036018640 [label=ToCopyBackward0]
140193036005136 -> 140193036018640
140193036005136 [label=SliceBackward0]
140193036018784 -> 140193036005136
140193036018784 [label=SliceBackward0]
140193036018880 -> 140193036018784
140193036018880 [label=NativeLayerNormBackward0]
140193036018976 -> 140193036018880
140193036018976 [label=AddBackward0]
140193036019168 -> 140193036018976
140193036019168 [label=NativeDropoutBackward0]
140193036019312 -> 140193036019168
140193036019312 [label=ViewBackward0]
140193036019408 -> 140193036019312
140193036019408 [label=AddmmBackward0]
140193036019504 -> 140193036019408
140193036019504 [label=ToCopyBackward0]
140193036019696 -> 140193036019504
140193039579952 [label="encoder.layer.0.crossattention.output.dense.bias
(768)" fillcolor=lightblue]
140193039579952 -> 140193036019696
140193036019696 [label=AccumulateGrad]
140193036019456 -> 140193036019408
140193036019456 [label=ViewBackward0]
140193036019744 -> 140193036019456
140193036019744 [label=ViewBackward0]
140193036019840 -> 140193036019744
140193036019840 [label=CloneBackward0]
140193036019936 -> 140193036019840
140193036019936 [label=PermuteBackward0]
140193036020032 -> 140193036019936
140193036020032 [label=UnsafeViewBackward0]
140193036020128 -> 140193036020032
140193036020128 [label=BmmBackward0]
140193036020224 -> 140193036020128
140193036020224 [label=ReshapeAliasBackward0]
140193036020368 -> 140193036020224
140193036020368 [label=ExpandBackward0]
140193036020464 -> 140193036020368
140193036020464 [label=ToCopyBackward0]
140193036020560 -> 140193036020464
140193036020560 [label=NativeDropoutBackward0]
140193036020656 -> 140193036020560
140193036020656 [label=SoftmaxBackward0]
140193036020752 -> 140193036020656
140193036020752 [label=AddBackward0]
140193036020848 -> 140193036020752
140193036020848 [label=DivBackward0]
140193036020944 -> 140193036020848
140193036020944 [label=UnsafeViewBackward0]
140193036021040 -> 140193036020944
140193036021040 [label=BmmBackward0]
140193036021136 -> 140193036021040
140193036021136 [label=UnsafeViewBackward0]
140193036021280 -> 140193036021136
140193036021280 [label=CloneBackward0]
140193036021376 -> 140193036021280
140193036021376 [label=ExpandBackward0]
140193036021472 -> 140193036021376
140193036021472 [label=PermuteBackward0]
140193036021568 -> 140193036021472
140193036021568 [label=ViewBackward0]
140193036021664 -> 140193036021568
140193036021664 [label=ViewBackward0]
140193036021712 -> 140193036021664
140193036021712 [label=AddmmBackward0]
140193036034208 -> 140193036021712
140193036034208 [label=ToCopyBackward0]
140193036034400 -> 140193036034208
140193039580672 [label="encoder.layer.0.crossattention.self.query.bias
(768)" fillcolor=lightblue]
140193039580672 -> 140193036034400
140193036034400 [label=AccumulateGrad]
140193036034160 -> 140193036021712
140193036034160 [label=ViewBackward0]
140193036034448 -> 140193036034160
140193036034448 [label=ToCopyBackward0]
140193036019120 -> 140193036034448
140193036019120 [label=SliceBackward0]
140193036034592 -> 140193036019120
140193036034592 [label=SliceBackward0]
140193036034688 -> 140193036034592
140193036034688 [label=SliceBackward0]
140193036034784 -> 140193036034688
140193036034784 [label=NativeLayerNormBackward0]
140193036034880 -> 140193036034784
140193036034880 [label=AddBackward0]
140193036035072 -> 140193036034880
140193036035072 [label=NativeDropoutBackward0]
140193036035216 -> 140193036035072
140193036035216 [label=ViewBackward0]
140193036035312 -> 140193036035216
140193036035312 [label=AddmmBackward0]
140193036035408 -> 140193036035312
140193036035408 [label=ToCopyBackward0]
140193036035600 -> 140193036035408
140193039581072 [label="encoder.layer.0.attention.output.dense.bias
(768)" fillcolor=lightblue]
140193039581072 -> 140193036035600
140193036035600 [label=AccumulateGrad]
140193036035360 -> 140193036035312
140193036035360 [label=ViewBackward0]
140193036035648 -> 140193036035360
140193036035648 [label=ViewBackward0]
140193036035744 -> 140193036035648
140193036035744 [label=CloneBackward0]
140193036035840 -> 140193036035744
140193036035840 [label=PermuteBackward0]
140193036035936 -> 140193036035840
140193036035936 [label=UnsafeViewBackward0]
140193036036032 -> 140193036035936
140193036036032 [label=BmmBackward0]
140193036036128 -> 140193036036032
140193036036128 [label=ReshapeAliasBackward0]
140193036036272 -> 140193036036128
140193036036272 [label=ExpandBackward0]
140193036036368 -> 140193036036272
140193036036368 [label=ToCopyBackward0]
140193036036464 -> 140193036036368
140193036036464 [label=NativeDropoutBackward0]
140193036036560 -> 140193036036464
140193036036560 [label=SoftmaxBackward0]
140193036036656 -> 140193036036560
140193036036656 [label=AddBackward0]
140193036036752 -> 140193036036656
140193036036752 [label=DivBackward0]
140193036036848 -> 140193036036752
140193036036848 [label=UnsafeViewBackward0]
140193036036944 -> 140193036036848
140193036036944 [label=BmmBackward0]
140193036037040 -> 140193036036944
140193036037040 [label=UnsafeViewBackward0]
140193036037184 -> 140193036037040
140193036037184 [label=CloneBackward0]
140193036037280 -> 140193036037184
140193036037280 [label=ExpandBackward0]
140193036037376 -> 140193036037280
140193036037376 [label=PermuteBackward0]
140193036037472 -> 140193036037376
140193036037472 [label=ViewBackward0]
140193036037568 -> 140193036037472
140193036037568 [label=ViewBackward0]
140193036037664 -> 140193036037568
140193036037664 [label=AddmmBackward0]
140193036037760 -> 140193036037664
140193036037760 [label=ToCopyBackward0]
140193036037952 -> 140193036037760
140193039248416 [label="encoder.layer.0.attention.self.query.bias
(768)" fillcolor=lightblue]
140193039248416 -> 140193036037952
140193036037952 [label=AccumulateGrad]
140193036037712 -> 140193036037664
140193036037712 [label=ViewBackward0]
140193036038000 -> 140193036037712
140193036038000 [label=ToCopyBackward0]
140193036035024 -> 140193036038000
140193036035024 [label=NativeDropoutBackward0]
140193036037904 -> 140193036035024
140193036037904 [label=NativeLayerNormBackward0]
140193036046400 -> 140193036037904
140193036046400 [label=CatBackward0]
140193036046688 -> 140193036046400
140193036046688 [label=ExpandBackward0]
140193036046832 -> 140193036046688
140194225446800 [label="
(1, 32, 768)" fillcolor=lightblue]
140194225446800 -> 140193036046832
140193036046832 [label=AccumulateGrad]
140193036046640 -> 140193036046400
140193036046640 [label=AddBackward0]
140193036046880 -> 140193036046640
140193036046880 [label=EmbeddingBackward0]
140193036047024 -> 140193036046880
140193039591120 [label="embeddings.word_embeddings.weight
(30523, 768)" fillcolor=lightblue]
140193039591120 -> 140193036047024
140193036047024 [label=AccumulateGrad]
140193036046928 -> 140193036046640
140193036046928 [label=EmbeddingBackward0]
140193036047072 -> 140193036046928
140194041968896 [label="embeddings.position_embeddings.weight
(512, 768)" fillcolor=lightblue]
140194041968896 -> 140193036047072
140193036047072 [label=AccumulateGrad]
140193036046448 -> 140193036037904
140193039245376 [label="embeddings.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039245376 -> 140193036046448
140193036046448 [label=AccumulateGrad]
140193036046496 -> 140193036037904
140193039605824 [label="embeddings.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039605824 -> 140193036046496
140193036046496 [label=AccumulateGrad]
140193036037088 -> 140193036037664
140193036037088 [label=TBackward0]
140193036037856 -> 140193036037088
140193036037856 [label=ToCopyBackward0]
140193036038048 -> 140193036037856
140193039247696 [label="encoder.layer.0.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039247696 -> 140193036038048
140193036038048 [label=AccumulateGrad]
140193036036992 -> 140193036036944
140193036036992 [label=UnsafeViewBackward0]
140193036037328 -> 140193036036992
140193036037328 [label=CloneBackward0]
140193036037520 -> 140193036037328
140193036037520 [label=ExpandBackward0]
140193036037808 -> 140193036037520
140193036037808 [label=TransposeBackward0]
140193036038096 -> 140193036037808
140193036038096 [label=PermuteBackward0]
140193036046784 -> 140193036038096
140193036046784 [label=ViewBackward0]
140193036047168 -> 140193036046784
140193036047168 [label=ViewBackward0]
140193036046976 -> 140193036047168
140193036046976 [label=AddmmBackward0]
140193036047264 -> 140193036046976
140193036047264 [label=ToCopyBackward0]
140193036047456 -> 140193036047264
140193039589920 [label="encoder.layer.0.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039589920 -> 140193036047456
140193036047456 [label=AccumulateGrad]
140193036047216 -> 140193036046976
140193036047216 [label=ViewBackward0]
140193036047504 -> 140193036047216
140193036047504 [label=ToCopyBackward0]
140193036035024 -> 140193036047504
140193036046592 -> 140193036046976
140193036046592 [label=TBackward0]
140193036047360 -> 140193036046592
140193036047360 [label=ToCopyBackward0]
140193036047648 -> 140193036047360
140193039589840 [label="encoder.layer.0.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039589840 -> 140193036047648
140193036047648 [label=AccumulateGrad]
140193036036080 -> 140193036036032
140193036036080 [label=UnsafeViewBackward0]
140193036036416 -> 140193036036080
140193036036416 [label=CloneBackward0]
140193036036608 -> 140193036036416
140193036036608 [label=ExpandBackward0]
140193036036800 -> 140193036036608
140193036036800 [label=PermuteBackward0]
140193036036176 -> 140193036036800
140193036036176 [label=ViewBackward0]
140193036037424 -> 140193036036176
140193036037424 [label=ViewBackward0]
140193036037136 -> 140193036037424
140193036037136 [label=AddmmBackward0]
140193036036224 -> 140193036037136
140193036036224 [label=ToCopyBackward0]
140193036047408 -> 140193036036224
140193039589680 [label="encoder.layer.0.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039589680 -> 140193036047408
140193036047408 [label=AccumulateGrad]
140193036046544 -> 140193036037136
140193036046544 [label=ViewBackward0]
140193036047744 -> 140193036046544
140193036047744 [label=ToCopyBackward0]
140193036035024 -> 140193036047744
140193036047120 -> 140193036037136
140193036047120 [label=TBackward0]
140193036047312 -> 140193036047120
140193036047312 [label=ToCopyBackward0]
140193036047792 -> 140193036047312
140193039589600 [label="encoder.layer.0.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039589600 -> 140193036047792
140193036047792 [label=AccumulateGrad]
140193036035120 -> 140193036035312
140193036035120 [label=TBackward0]
140193036035792 -> 140193036035120
140193036035792 [label=ToCopyBackward0]
140193036035984 -> 140193036035792
140193039589440 [label="encoder.layer.0.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039589440 -> 140193036035984
140193036035984 [label=AccumulateGrad]
140193036035024 -> 140193036034880
140193036034832 -> 140193036034784
140193039580832 [label="encoder.layer.0.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039580832 -> 140193036034832
140193036034832 [label=AccumulateGrad]
140193036034304 -> 140193036034784
140193039580912 [label="encoder.layer.0.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039580912 -> 140193036034304
140193036034304 [label=AccumulateGrad]
140193036034112 -> 140193036021712
140193036034112 [label=TBackward0]
140193036034352 -> 140193036034112
140193036034352 [label=ToCopyBackward0]
140193036034736 -> 140193036034352
140193039580592 [label="encoder.layer.0.crossattention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039580592 -> 140193036034736
140193036034736 [label=AccumulateGrad]
140193036021088 -> 140193036021040
140193036021088 [label=UnsafeViewBackward0]
140193036021424 -> 140193036021088
140193036021424 [label=CloneBackward0]
140193036021616 -> 140193036021424
140193036021616 [label=ExpandBackward0]
140193036021184 -> 140193036021616
140193036021184 [label=TransposeBackward0]
140193036034640 -> 140193036021184
140193036034640 [label=PermuteBackward0]
140193036034928 -> 140193036034640
140193036034928 [label=ViewBackward0]
140193036035168 -> 140193036034928
140193036035168 [label=ViewBackward0]
140193036035456 -> 140193036035168
140193036035456 [label=AddmmBackward0]
140193036035888 -> 140193036035456
140193036035888 [label=ToCopyBackward0]
140193036036512 -> 140193036035888
140193039580432 [label="encoder.layer.0.crossattention.self.key.bias
(768)" fillcolor=lightblue]
140193039580432 -> 140193036036512
140193036036512 [label=AccumulateGrad]
140193036035504 -> 140193036035456
140193036035504 [label=ViewBackward0]
140193036036704 -> 140193036035504
140193036036704 [label=ToCopyBackward0]
140193036037232 -> 140193036036704
140193036037232 [label=NativeLayerNormBackward0]
140193036037616 -> 140193036037232
140193039246816 [label="
(1408)" fillcolor=lightblue]
140193039246816 -> 140193036037616
140193036037616 [label=AccumulateGrad]
140193036035696 -> 140193036037232
140193039247056 [label="
(1408)" fillcolor=lightblue]
140193039247056 -> 140193036035696
140193036035696 [label=AccumulateGrad]
140193036034256 -> 140193036035456
140193036034256 [label=TBackward0]
140193036035552 -> 140193036034256
140193036035552 [label=ToCopyBackward0]
140193036047600 -> 140193036035552
140193039580352 [label="encoder.layer.0.crossattention.self.key.weight
(768, 1408)" fillcolor=lightblue]
140193039580352 -> 140193036047600
140193036047600 [label=AccumulateGrad]
140193036020176 -> 140193036020128
140193036020176 [label=UnsafeViewBackward0]
140193036020512 -> 140193036020176
140193036020512 [label=CloneBackward0]
140193036020704 -> 140193036020512
140193036020704 [label=ExpandBackward0]
140193036020896 -> 140193036020704
140193036020896 [label=PermuteBackward0]
140193036020272 -> 140193036020896
140193036020272 [label=ViewBackward0]
140193036021520 -> 140193036020272
140193036021520 [label=ViewBackward0]
140193036021232 -> 140193036021520
140193036021232 [label=AddmmBackward0]
140193036034496 -> 140193036021232
140193036034496 [label=ToCopyBackward0]
140193036036896 -> 140193036034496
140193039580192 [label="encoder.layer.0.crossattention.self.value.bias
(768)" fillcolor=lightblue]
140193039580192 -> 140193036036896
140193036036896 [label=AccumulateGrad]
140193036034976 -> 140193036021232
140193036034976 [label=ViewBackward0]
140193036036320 -> 140193036034976
140193036036320 [label=ToCopyBackward0]
140193036037232 -> 140193036036320
140193036034544 -> 140193036021232
140193036034544 [label=TBackward0]
140193036047552 -> 140193036034544
140193036047552 [label=ToCopyBackward0]
140193036047696 -> 140193036047552
140193039580112 [label="encoder.layer.0.crossattention.self.value.weight
(768, 1408)" fillcolor=lightblue]
140193039580112 -> 140193036047696
140193036047696 [label=AccumulateGrad]
140193036019216 -> 140193036019408
140193036019216 [label=TBackward0]
140193036019888 -> 140193036019216
140193036019888 [label=ToCopyBackward0]
140193036020080 -> 140193036019888
140193039579872 [label="encoder.layer.0.crossattention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039579872 -> 140193036020080
140193036020080 [label=AccumulateGrad]
140193036019120 -> 140193036018976
140193036018928 -> 140193036018880
140193039579632 [label="encoder.layer.0.crossattention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039579632 -> 140193036018928
140193036018928 [label=AccumulateGrad]
140193036018496 -> 140193036018880
140193039579712 [label="encoder.layer.0.crossattention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039579712 -> 140193036018496
140193036018496 [label=AccumulateGrad]
140193036018016 -> 140193036018304
140193036018016 [label=TBackward0]
140193036018544 -> 140193036018016
140193036018544 [label=ToCopyBackward0]
140193036019024 -> 140193036018544
140194226510320 [label="encoder.layer.0.experts.dense1.weight
(3072, 768)" fillcolor=lightblue]
140194226510320 -> 140193036019024
140193036019024 [label=AccumulateGrad]
140193036017728 -> 140193036005232
140193036017728 [label=TBackward0]
140193036018256 -> 140193036017728
140193036018256 [label=ToCopyBackward0]
140193036018736 -> 140193036018256
140193039578032 [label="encoder.layer.0.experts.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039578032 -> 140193036018736
140193036018736 [label=AccumulateGrad]
140193036005136 -> 140193036004992
140193036004944 -> 140193036004848
140193039577792 [label="encoder.layer.0.expert_ln.weight
(768)" fillcolor=lightblue]
140193039577792 -> 140193036004944
140193036004944 [label=AccumulateGrad]
140193036004896 -> 140193036004848
140193039577472 [label="encoder.layer.0.expert_ln.bias
(768)" fillcolor=lightblue]
140193039577472 -> 140193036004896
140193036004896 [label=AccumulateGrad]
140193036004608 -> 140193036001728
140193036004608 [label=NativeLayerNormBackward0]
140193036005280 -> 140193036004608
140193036005280 [label=AddBackward0]
140193036018448 -> 140193036005280
140193036018448 [label=NativeDropoutBackward0]
140193036018160 -> 140193036018448
140193036018160 [label=ViewBackward0]
140193036018688 -> 140193036018160
140193036018688 [label=AddmmBackward0]
140193036019552 -> 140193036018688
140193036019552 [label=ToCopyBackward0]
140193036019648 -> 140193036019552
140193039579232 [label="encoder.layer.0.output.dense.bias
(768)" fillcolor=lightblue]
140193039579232 -> 140193036019648
140193036019648 [label=AccumulateGrad]
140193036019360 -> 140193036018688
140193036019360 [label=ViewBackward0]
140193036019792 -> 140193036019360
140193036019792 [label=GeluBackward0]
140193036020800 -> 140193036019792
140193036020800 [label=ViewBackward0]
140193036021328 -> 140193036020800
140193036021328 [label=AddmmBackward0]
140193036035264 -> 140193036021328
140193036035264 [label=ToCopyBackward0]
140193036047936 -> 140193036035264
140193039579472 [label="encoder.layer.0.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039579472 -> 140193036047936
140193036047936 [label=AccumulateGrad]
140193036020320 -> 140193036021328
140193036020320 [label=ViewBackward0]
140193036047840 -> 140193036020320
140193036047840 [label=ToCopyBackward0]
140193036017968 -> 140193036047840
140193036017968 [label=SliceBackward0]
140193036048128 -> 140193036017968
140193036048128 [label=SliceBackward0]
140193036048224 -> 140193036048128
140193036048224 [label=SliceBackward0]
140193036034784 -> 140193036048224
140193036020416 -> 140193036021328
140193036020416 [label=TBackward0]
140193036047984 -> 140193036020416
140193036047984 [label=ToCopyBackward0]
140193036048320 -> 140193036047984
140193039579392 [label="encoder.layer.0.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039579392 -> 140193036048320
140193036048320 [label=AccumulateGrad]
140193036019264 -> 140193036018688
140193036019264 [label=TBackward0]
140193036020992 -> 140193036019264
140193036020992 [label=ToCopyBackward0]
140193036020608 -> 140193036020992
140193039579152 [label="encoder.layer.0.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039579152 -> 140193036020608
140193036020608 [label=AccumulateGrad]
140193036017968 -> 140193036005280
140193036005088 -> 140193036004608
140193039578912 [label="encoder.layer.0.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039578912 -> 140193036005088
140193036005088 [label=AccumulateGrad]
140193036005040 -> 140193036004608
140193039578992 [label="encoder.layer.0.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039578992 -> 140193036005040
140193036005040 [label=AccumulateGrad]
140193036003792 -> 140193036004368
140193036003792 [label=TBackward0]
140193036004560 -> 140193036003792
140193036004560 [label=ToCopyBackward0]
140193036004752 -> 140193036004560
140193039577552 [label="encoder.layer.1.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039577552 -> 140193036004752
140193036004752 [label=AccumulateGrad]
140193036003696 -> 140193036003648
140193036003696 [label=UnsafeViewBackward0]
140193036004032 -> 140193036003696
140193036004032 [label=CloneBackward0]
140193036004224 -> 140193036004032
140193036004224 [label=ExpandBackward0]
140193036004512 -> 140193036004224
140193036004512 [label=TransposeBackward0]
140193036004800 -> 140193036004512
140193036004800 [label=PermuteBackward0]
140193036018832 -> 140193036004800
140193036018832 [label=ViewBackward0]
140193036019600 -> 140193036018832
140193036019600 [label=ViewBackward0]
140193036019984 -> 140193036019600
140193036019984 [label=AddmmBackward0]
140193036048176 -> 140193036019984
140193036048176 [label=ToCopyBackward0]
140193036048368 -> 140193036048176
140193039560512 [label="encoder.layer.1.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039560512 -> 140193036048368
140193036048368 [label=AccumulateGrad]
140193036047888 -> 140193036019984
140193036047888 [label=ViewBackward0]
140193036048416 -> 140193036047888
140193036048416 [label=ToCopyBackward0]
140193036001728 -> 140193036048416
140193036048080 -> 140193036019984
140193036048080 [label=TBackward0]
140193036048272 -> 140193036048080
140193036048272 [label=ToCopyBackward0]
140193036048560 -> 140193036048272
140193039577312 [label="encoder.layer.1.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039577312 -> 140193036048560
140193036048560 [label=AccumulateGrad]
140193036002784 -> 140193036002736
140193036002784 [label=UnsafeViewBackward0]
140193036003120 -> 140193036002784
140193036003120 [label=CloneBackward0]
140193036003312 -> 140193036003120
140193036003312 [label=ExpandBackward0]
140193036003504 -> 140193036003312
140193036003504 [label=PermuteBackward0]
140193036002880 -> 140193036003504
140193036002880 [label=ViewBackward0]
140193036004128 -> 140193036002880
140193036004128 [label=ViewBackward0]
140193036003840 -> 140193036004128
140193036003840 [label=AddmmBackward0]
140193036002928 -> 140193036003840
140193036002928 [label=ToCopyBackward0]
140193036048032 -> 140193036002928
140193039560272 [label="encoder.layer.1.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039560272 -> 140193036048032
140193036048032 [label=AccumulateGrad]
140193036017776 -> 140193036003840
140193036017776 [label=ViewBackward0]
140193036048656 -> 140193036017776
140193036048656 [label=ToCopyBackward0]
140193036001728 -> 140193036048656
140193036019072 -> 140193036003840
140193036019072 [label=TBackward0]
140193036046736 -> 140193036019072
140193036046736 [label=ToCopyBackward0]
140193036048704 -> 140193036046736
140193039560592 [label="encoder.layer.1.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039560592 -> 140193036048704
140193036048704 [label=AccumulateGrad]
140193036001824 -> 140193036002016
140193036001824 [label=TBackward0]
140193036002496 -> 140193036001824
140193036002496 [label=ToCopyBackward0]
140193036002688 -> 140193036002496
140193039560352 [label="encoder.layer.1.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039560352 -> 140193036002688
140193036002688 [label=AccumulateGrad]
140193036001728 -> 140193036001584
140193036001536 -> 140193036001488
140193039560112 [label="encoder.layer.1.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039560112 -> 140193036001536
140193036001536 [label=AccumulateGrad]
140193036001344 -> 140193036001488
140193039559792 [label="encoder.layer.1.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039559792 -> 140193036001344
140193036001344 [label=AccumulateGrad]
140193035987984 -> 140193035988272
140193035987984 [label=TBackward0]
140193035988512 -> 140193035987984
140193035988512 [label=ToCopyBackward0]
140193035988896 -> 140193035988512
140193039558432 [label="encoder.layer.1.experts.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039558432 -> 140193035988896
140193035988896 [label=AccumulateGrad]
140193035987552 -> 140193035987744
140193035987552 [label=TBackward0]
140193035988224 -> 140193035987552
140193035988224 [label=ToCopyBackward0]
140193035988704 -> 140193035988224
140193039558192 [label="encoder.layer.1.experts.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039558192 -> 140193035988704
140193035988704 [label=AccumulateGrad]
140193035987456 -> 140193035987312
140193035987264 -> 140193035987168
140193039557952 [label="encoder.layer.1.expert_ln.weight
(768)" fillcolor=lightblue]
140193039557952 -> 140193035987264
140193035987264 [label=AccumulateGrad]
140193035987216 -> 140193035987168
140193039557632 [label="encoder.layer.1.expert_ln.bias
(768)" fillcolor=lightblue]
140193039557632 -> 140193035987216
140193035987216 [label=AccumulateGrad]
140193035986928 -> 140193035963504
140193035986928 [label=NativeLayerNormBackward0]
140193035987600 -> 140193035986928
140193035987600 [label=AddBackward0]
140193035988416 -> 140193035987600
140193035988416 [label=NativeDropoutBackward0]
140193035988128 -> 140193035988416
140193035988128 [label=ViewBackward0]
140193035988656 -> 140193035988128
140193035988656 [label=AddmmBackward0]
140193036001680 -> 140193035988656
140193036001680 [label=ToCopyBackward0]
140193036002208 -> 140193036001680
140193039559312 [label="encoder.layer.1.output.dense.bias
(768)" fillcolor=lightblue]
140193039559312 -> 140193036002208
140193036002208 [label=AccumulateGrad]
140193036001632 -> 140193035988656
140193036001632 [label=ViewBackward0]
140193036002592 -> 140193036001632
140193036002592 [label=GeluBackward0]
140193036002256 -> 140193036002592
140193036002256 [label=ViewBackward0]
140193036003216 -> 140193036002256
140193036003216 [label=AddmmBackward0]
140193036003600 -> 140193036003216
140193036003600 [label=ToCopyBackward0]
140193036017920 -> 140193036003600
140193039559552 [label="encoder.layer.1.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039559552 -> 140193036017920
140193036017920 [label=AccumulateGrad]
140193036003408 -> 140193036003216
140193036003408 [label=ViewBackward0]
140193036004320 -> 140193036003408
140193036004320 [label=ToCopyBackward0]
140193035987936 -> 140193036004320
140193035987936 [label=SliceBackward0]
140193036048608 -> 140193035987936
140193036048608 [label=SliceBackward0]
140193036048896 -> 140193036048608
140193036048896 [label=SliceBackward0]
140193036001488 -> 140193036048896
140193036002160 -> 140193036003216
140193036002160 [label=TBackward0]
140193036048800 -> 140193036002160
140193036048800 [label=ToCopyBackward0]
140193036048992 -> 140193036048800
140193039559872 [label="encoder.layer.1.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039559872 -> 140193036048992
140193036048992 [label=AccumulateGrad]
140193036001440 -> 140193035988656
140193036001440 [label=TBackward0]
140193036002400 -> 140193036001440
140193036002400 [label=ToCopyBackward0]
140193036003936 -> 140193036002400
140193039559632 [label="encoder.layer.1.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039559632 -> 140193036003936
140193036003936 [label=AccumulateGrad]
140193035987936 -> 140193035987600
140193035987408 -> 140193035986928
140193039559392 [label="encoder.layer.1.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039559392 -> 140193035987408
140193035987408 [label=AccumulateGrad]
140193035987360 -> 140193035986928
140193039559072 [label="encoder.layer.1.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039559072 -> 140193035987360
140193035987360 [label=AccumulateGrad]
140193035986112 -> 140193035986688
140193035986112 [label=TBackward0]
140193035986880 -> 140193035986112
140193035986880 [label=ToCopyBackward0]
140193035987888 -> 140193035986880
140193039557712 [label="encoder.layer.2.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039557712 -> 140193035987888
140193035987888 [label=AccumulateGrad]
140193035986016 -> 140193035985968
140193035986016 [label=UnsafeViewBackward0]
140193035986352 -> 140193035986016
140193035986352 [label=CloneBackward0]
140193035986544 -> 140193035986352
140193035986544 [label=ExpandBackward0]
140193035986832 -> 140193035986544
140193035986832 [label=TransposeBackward0]
140193035987696 -> 140193035986832
140193035987696 [label=PermuteBackward0]
140193035987072 -> 140193035987696
140193035987072 [label=ViewBackward0]
140193035986160 -> 140193035987072
140193035986160 [label=ViewBackward0]
140193036003024 -> 140193035986160
140193036003024 [label=AddmmBackward0]
140193036001392 -> 140193036003024
140193036001392 [label=ToCopyBackward0]
140193036049040 -> 140193036001392
140193039557152 [label="encoder.layer.2.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039557152 -> 140193036049040
140193036049040 [label=AccumulateGrad]
140193036048848 -> 140193036003024
140193036048848 [label=ViewBackward0]
140193036049088 -> 140193036048848
140193036049088 [label=ToCopyBackward0]
140193035963504 -> 140193036049088
140193036048464 -> 140193036003024
140193036048464 [label=TBackward0]
140193036048944 -> 140193036048464
140193036048944 [label=ToCopyBackward0]
140193036049232 -> 140193036048944
140193039557472 [label="encoder.layer.2.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039557472 -> 140193036049232
140193036049232 [label=AccumulateGrad]
140193035985104 -> 140193035985056
140193035985104 [label=UnsafeViewBackward0]
140193035985440 -> 140193035985104
140193035985440 [label=CloneBackward0]
140193035985632 -> 140193035985440
140193035985632 [label=ExpandBackward0]
140193035985824 -> 140193035985632
140193035985824 [label=PermuteBackward0]
140193035985200 -> 140193035985824
140193035985200 [label=ViewBackward0]
140193035986448 -> 140193035985200
140193035986448 [label=ViewBackward0]
140193035987120 -> 140193035986448
140193035987120 [label=AddmmBackward0]
140193035988800 -> 140193035987120
140193035988800 [label=ToCopyBackward0]
140193036048512 -> 140193035988800
140193039556912 [label="encoder.layer.2.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039556912 -> 140193036048512
140193036048512 [label=AccumulateGrad]
140193035985248 -> 140193035987120
140193035985248 [label=ViewBackward0]
140193036049328 -> 140193035985248
140193036049328 [label=ToCopyBackward0]
140193035963504 -> 140193036049328
140193036001872 -> 140193035987120
140193036001872 [label=TBackward0]
140193036048752 -> 140193036001872
140193036048752 [label=ToCopyBackward0]
140193036049376 -> 140193036048752
140193039557232 [label="encoder.layer.2.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039557232 -> 140193036049376
140193036049376 [label=AccumulateGrad]
140193035963600 -> 140193035963792
140193035963600 [label=TBackward0]
140193035964272 -> 140193035963600
140193035964272 [label=ToCopyBackward0]
140193035964032 -> 140193035964272
140193039556992 [label="encoder.layer.2.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039556992 -> 140193035964032
140193035964032 [label=AccumulateGrad]
140193035963504 -> 140193035963360
140193035963312 -> 140193035963264
140193039556752 [label="encoder.layer.2.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039556752 -> 140193035963312
140193035963312 [label=AccumulateGrad]
140193035962784 -> 140193035963264
140193039552240 [label="encoder.layer.2.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039552240 -> 140193035962784
140193035962784 [label=AccumulateGrad]
140193035962016 -> 140193035962592
140193035962016 [label=TBackward0]
140193035962832 -> 140193035962016
140193035962832 [label=ToCopyBackward0]
140193035963216 -> 140193035962832
140193039552320 [label="encoder.layer.2.crossattention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039552320 -> 140193035963216
140193035963216 [label=AccumulateGrad]
140193035961920 -> 140193035961872
140193035961920 [label=UnsafeViewBackward0]
140193035962256 -> 140193035961920
140193035962256 [label=CloneBackward0]
140193035962448 -> 140193035962256
140193035962448 [label=ExpandBackward0]
140193035962736 -> 140193035962448
140193035962736 [label=TransposeBackward0]
140193035963120 -> 140193035962736
140193035963120 [label=PermuteBackward0]
140193035963408 -> 140193035963120
140193035963408 [label=ViewBackward0]
140193035963648 -> 140193035963408
140193035963648 [label=ViewBackward0]
140193035963936 -> 140193035963648
140193035963936 [label=AddmmBackward0]
140193035964176 -> 140193035963936
140193035964176 [label=ToCopyBackward0]
140193035985536 -> 140193035964176
140193039551760 [label="encoder.layer.2.crossattention.self.key.bias
(768)" fillcolor=lightblue]
140193039551760 -> 140193035985536
140193035985536 [label=AccumulateGrad]
140193035963984 -> 140193035963936
140193035963984 [label=ViewBackward0]
140193035985728 -> 140193035963984
140193035985728 [label=ToCopyBackward0]
140193036037232 -> 140193035985728
140193035962064 -> 140193035963936
140193035962064 [label=TBackward0]
140193035984960 -> 140193035962064
140193035984960 [label=ToCopyBackward0]
140193035986640 -> 140193035984960
140193039552080 [label="encoder.layer.2.crossattention.self.key.weight
(768, 1408)" fillcolor=lightblue]
140193039552080 -> 140193035986640
140193035986640 [label=AccumulateGrad]
140193035961008 -> 140193035960960
140193035961008 [label=UnsafeViewBackward0]
140193035961344 -> 140193035961008
140193035961344 [label=CloneBackward0]
140193035961536 -> 140193035961344
140193035961536 [label=ExpandBackward0]
140193035961728 -> 140193035961536
140193035961728 [label=PermuteBackward0]
140193035961104 -> 140193035961728
140193035961104 [label=ViewBackward0]
140193035962352 -> 140193035961104
140193035962352 [label=ViewBackward0]
140193035963024 -> 140193035962352
140193035963024 [label=AddmmBackward0]
140193035962976 -> 140193035963024
140193035962976 [label=ToCopyBackward0]
140193036001968 -> 140193035962976
140193039551520 [label="encoder.layer.2.crossattention.self.value.bias
(768)" fillcolor=lightblue]
140193039551520 -> 140193036001968
140193036001968 [label=AccumulateGrad]
140193035963456 -> 140193035963024
140193035963456 [label=ViewBackward0]
140193035985008 -> 140193035963456
140193035985008 [label=ToCopyBackward0]
140193036037232 -> 140193035985008
140193035961152 -> 140193035963024
140193035961152 [label=TBackward0]
140193035986256 -> 140193035961152
140193035986256 [label=ToCopyBackward0]
140193035985920 -> 140193035986256
140193039551840 [label="encoder.layer.2.crossattention.self.value.weight
(768, 1408)" fillcolor=lightblue]
140193039551840 -> 140193035985920
140193035985920 [label=AccumulateGrad]
140193035955888 -> 140193035956080
140193035955888 [label=TBackward0]
140193035960720 -> 140193035955888
140193035960720 [label=ToCopyBackward0]
140193035960912 -> 140193035960720
140193039551600 [label="encoder.layer.2.crossattention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039551600 -> 140193035960912
140193035960912 [label=AccumulateGrad]
140193035955792 -> 140193035955648
140193035955600 -> 140193035955552
140193039551360 [label="encoder.layer.2.crossattention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039551360 -> 140193035955600
140193035955600 [label=AccumulateGrad]
140193035955168 -> 140193035955552
140193039551040 [label="encoder.layer.2.crossattention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039551040 -> 140193035955168
140193035955168 [label=AccumulateGrad]
140193035954688 -> 140193035954976
140193035954688 [label=TBackward0]
140193035955216 -> 140193035954688
140193035955216 [label=ToCopyBackward0]
140193035955696 -> 140193035955216
140193039549680 [label="encoder.layer.2.experts.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039549680 -> 140193035955696
140193035955696 [label=AccumulateGrad]
140193035954256 -> 140193035954448
140193035954256 [label=TBackward0]
140193035954928 -> 140193035954256
140193035954928 [label=ToCopyBackward0]
140193035955408 -> 140193035954928
140193039549440 [label="encoder.layer.2.experts.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039549440 -> 140193035955408
140193035955408 [label=AccumulateGrad]
140193035954160 -> 140193035954016
140193035953968 -> 140193035953872
140193039549200 [label="encoder.layer.2.expert_ln.weight
(768)" fillcolor=lightblue]
140193039549200 -> 140193035953968
140193035953968 [label=AccumulateGrad]
140193035953920 -> 140193035953872
140193039548880 [label="encoder.layer.2.expert_ln.bias
(768)" fillcolor=lightblue]
140193039548880 -> 140193035953920
140193035953920 [label=AccumulateGrad]
140193035953632 -> 140193037261408
140193035953632 [label=NativeLayerNormBackward0]
140193035954304 -> 140193035953632
140193035954304 [label=AddBackward0]
140193035955120 -> 140193035954304
140193035955120 [label=NativeDropoutBackward0]
140193035954832 -> 140193035955120
140193035954832 [label=ViewBackward0]
140193035955360 -> 140193035954832
140193035955360 [label=AddmmBackward0]
140193035956032 -> 140193035955360
140193035956032 [label=ToCopyBackward0]
140193035960480 -> 140193035956032
140193039550560 [label="encoder.layer.2.output.dense.bias
(768)" fillcolor=lightblue]
140193039550560 -> 140193035960480
140193035960480 [label=AccumulateGrad]
140193035955936 -> 140193035955360
140193035955936 [label=ViewBackward0]
140193035960624 -> 140193035955936
140193035960624 [label=GeluBackward0]
140193035961632 -> 140193035960624
140193035961632 [label=ViewBackward0]
140193035962160 -> 140193035961632
140193035962160 [label=AddmmBackward0]
140193035963744 -> 140193035962160
140193035963744 [label=ToCopyBackward0]
140193036049472 -> 140193035963744
140193039550800 [label="encoder.layer.2.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039550800 -> 140193036049472
140193036049472 [label=AccumulateGrad]
140193035962544 -> 140193035962160
140193035962544 [label=ViewBackward0]
140193036049136 -> 140193035962544
140193036049136 [label=ToCopyBackward0]
140193035954640 -> 140193036049136
140193035954640 [label=SliceBackward0]
140193036049568 -> 140193035954640
140193036049568 [label=SliceBackward0]
140193036049664 -> 140193036049568
140193036049664 [label=SliceBackward0]
140193035963264 -> 140193036049664
140193035961248 -> 140193035962160
140193035961248 [label=TBackward0]
140193036049424 -> 140193035961248
140193036049424 [label=ToCopyBackward0]
140193036049760 -> 140193036049424
140193039551120 [label="encoder.layer.2.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039551120 -> 140193036049760
140193036049760 [label=AccumulateGrad]
140193035960384 -> 140193035955360
140193035960384 [label=TBackward0]
140193035961824 -> 140193035960384
140193035961824 [label=ToCopyBackward0]
140193035985344 -> 140193035961824
140193039550880 [label="encoder.layer.2.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039550880 -> 140193035985344
140193035985344 [label=AccumulateGrad]
140193035954640 -> 140193035954304
140193035954112 -> 140193035953632
140193039550640 [label="encoder.layer.2.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039550640 -> 140193035954112
140193035954112 [label=AccumulateGrad]
140193035954064 -> 140193035953632
140193039550320 [label="encoder.layer.2.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039550320 -> 140193035954064
140193035954064 [label=AccumulateGrad]
140193035952816 -> 140193035953392
140193035952816 [label=TBackward0]
140193035953584 -> 140193035952816
140193035953584 [label=ToCopyBackward0]
140193035954592 -> 140193035953584
140193039548960 [label="encoder.layer.3.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039548960 -> 140193035954592
140193035954592 [label=AccumulateGrad]
140193035952720 -> 140193035952672
140193035952720 [label=UnsafeViewBackward0]
140193035953056 -> 140193035952720
140193035953056 [label=CloneBackward0]
140193035953248 -> 140193035953056
140193035953248 [label=ExpandBackward0]
140193035953536 -> 140193035953248
140193035953536 [label=TransposeBackward0]
140193035954400 -> 140193035953536
140193035954400 [label=PermuteBackward0]
140193035955504 -> 140193035954400
140193035955504 [label=ViewBackward0]
140193035953776 -> 140193035955504
140193035953776 [label=ViewBackward0]
140193035961440 -> 140193035953776
140193035961440 [label=AddmmBackward0]
140193035960432 -> 140193035961440
140193035960432 [label=ToCopyBackward0]
140193036049808 -> 140193035960432
140193039548480 [label="encoder.layer.3.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039548480 -> 140193036049808
140193036049808 [label=AccumulateGrad]
140193036049616 -> 140193035961440
140193036049616 [label=ViewBackward0]
140193036049856 -> 140193036049616
140193036049856 [label=ToCopyBackward0]
140193037261408 -> 140193036049856
140193036049184 -> 140193035961440
140193036049184 [label=TBackward0]
140193036049712 -> 140193036049184
140193036049712 [label=ToCopyBackward0]
140193036050000 -> 140193036049712
140193039548720 [label="encoder.layer.3.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039548720 -> 140193036050000
140193036050000 [label=AccumulateGrad]
140193037262464 -> 140193037262416
140193037262464 [label=UnsafeViewBackward0]
140193037262560 -> 140193037262464
140193037262560 [label=CloneBackward0]
140193037262608 -> 140193037262560
140193037262608 [label=ExpandBackward0]
140193035952528 -> 140193037262608
140193035952528 [label=PermuteBackward0]
140193035952192 -> 140193035952528
140193035952192 [label=ViewBackward0]
140193035953152 -> 140193035952192
140193035953152 [label=ViewBackward0]
140193035953824 -> 140193035953152
140193035953824 [label=AddmmBackward0]
140193035952864 -> 140193035953824
140193035952864 [label=ToCopyBackward0]
140193036049280 -> 140193035952864
140193039539872 [label="encoder.layer.3.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039539872 -> 140193036049280
140193036049280 [label=AccumulateGrad]
140193035955744 -> 140193035953824
140193035955744 [label=ViewBackward0]
140193036050096 -> 140193035955744
140193036050096 [label=ToCopyBackward0]
140193037261408 -> 140193036050096
140193035952240 -> 140193035953824
140193035952240 [label=TBackward0]
140193036049520 -> 140193035952240
140193036049520 [label=ToCopyBackward0]
140193036050144 -> 140193036049520
140193039540112 [label="encoder.layer.3.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039540112 -> 140193036050144
140193036050144 [label=AccumulateGrad]
140193037261504 -> 140193037261696
140193037261504 [label=TBackward0]
140193037262176 -> 140193037261504
140193037262176 [label=ToCopyBackward0]
140193037262368 -> 140193037262176
140193039539952 [label="encoder.layer.3.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039539952 -> 140193037262368
140193037262368 [label=AccumulateGrad]
140193037261408 -> 140193037261264
140193037261216 -> 140193037261168
140193039539712 [label="encoder.layer.3.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039539712 -> 140193037261216
140193037261216 [label=AccumulateGrad]
140193037260496 -> 140193037261168
140193039539392 [label="encoder.layer.3.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039539392 -> 140193037260496
140193037260496 [label=AccumulateGrad]
140193037260016 -> 140193037260304
140193037260016 [label=TBackward0]
140193037260544 -> 140193037260016
140193037260544 [label=ToCopyBackward0]
140193037260928 -> 140193037260544
140193039538032 [label="encoder.layer.3.experts.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039538032 -> 140193037260928
140193037260928 [label=AccumulateGrad]
140193037259584 -> 140193037259776
140193037259584 [label=TBackward0]
140193037260256 -> 140193037259584
140193037260256 [label=ToCopyBackward0]
140193037260736 -> 140193037260256
140193039537792 [label="encoder.layer.3.experts.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039537792 -> 140193037260736
140193037260736 [label=AccumulateGrad]
140193037259488 -> 140193037259344
140193037259296 -> 140193037259200
140193039537552 [label="encoder.layer.3.expert_ln.weight
(768)" fillcolor=lightblue]
140193039537552 -> 140193037259296
140193037259296 [label=AccumulateGrad]
140193037259248 -> 140193037259200
140193039537232 [label="encoder.layer.3.expert_ln.bias
(768)" fillcolor=lightblue]
140193039537232 -> 140193037259248
140193037259248 [label=AccumulateGrad]
140193037258960 -> 140193037276496
140193037258960 [label=NativeLayerNormBackward0]
140193037259632 -> 140193037258960
140193037259632 [label=AddBackward0]
140193037260448 -> 140193037259632
140193037260448 [label=NativeDropoutBackward0]
140193037260160 -> 140193037260448
140193037260160 [label=ViewBackward0]
140193037260688 -> 140193037260160
140193037260688 [label=AddmmBackward0]
140193037261360 -> 140193037260688
140193037261360 [label=ToCopyBackward0]
140193037261888 -> 140193037261360
140193039538912 [label="encoder.layer.3.output.dense.bias
(768)" fillcolor=lightblue]
140193039538912 -> 140193037261888
140193037261888 [label=AccumulateGrad]
140193037261312 -> 140193037260688
140193037261312 [label=ViewBackward0]
140193037262272 -> 140193037261312
140193037262272 [label=GeluBackward0]
140193037261936 -> 140193037262272
140193037261936 [label=ViewBackward0]
140193037262080 -> 140193037261936
140193037262080 [label=AddmmBackward0]
140193035952624 -> 140193037262080
140193035952624 [label=ToCopyBackward0]
140193035960816 -> 140193035952624
140193039539152 [label="encoder.layer.3.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039539152 -> 140193035960816
140193035960816 [label=AccumulateGrad]
140193035952432 -> 140193037262080
140193035952432 [label=ViewBackward0]
140193035953344 -> 140193035952432
140193035953344 [label=ToCopyBackward0]
140193037259968 -> 140193035953344
140193037259968 [label=SliceBackward0]
140193036050048 -> 140193037259968
140193036050048 [label=SliceBackward0]
140193036050336 -> 140193036050048
140193036050336 [label=SliceBackward0]
140193037261168 -> 140193036050336
140193035952336 -> 140193037262080
140193035952336 [label=TBackward0]
140193036050240 -> 140193035952336
140193036050240 [label=ToCopyBackward0]
140193036050384 -> 140193036050240
140193039539472 [label="encoder.layer.3.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039539472 -> 140193036050384
140193036050384 [label=AccumulateGrad]
140193037261120 -> 140193037260688
140193037261120 [label=TBackward0]
140193037261840 -> 140193037261120
140193037261840 [label=ToCopyBackward0]
140193035952960 -> 140193037261840
140193039539232 [label="encoder.layer.3.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039539232 -> 140193035952960
140193035952960 [label=AccumulateGrad]
140193037259968 -> 140193037259632
140193037259440 -> 140193037258960
140193039538992 [label="encoder.layer.3.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039538992 -> 140193037259440
140193037259440 [label=AccumulateGrad]
140193037259392 -> 140193037258960
140193039538672 [label="encoder.layer.3.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039538672 -> 140193037259392
140193037259392 [label=AccumulateGrad]
140193037258816 -> 140193037279136
140193037258816 [label=TBackward0]
140193037258912 -> 140193037258816
140193037258912 [label=ToCopyBackward0]
140193037259920 -> 140193037258912
140193039537312 [label="encoder.layer.4.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039537312 -> 140193037259920
140193037259920 [label=AccumulateGrad]
140193037278464 -> 140193037278416
140193037278464 [label=UnsafeViewBackward0]
140193037278800 -> 140193037278464
140193037278800 [label=CloneBackward0]
140193037278992 -> 140193037278800
140193037278992 [label=ExpandBackward0]
140193037279088 -> 140193037278992
140193037279088 [label=TransposeBackward0]
140193037259728 -> 140193037279088
140193037259728 [label=PermuteBackward0]
140193037260832 -> 140193037259728
140193037260832 [label=ViewBackward0]
140193037261552 -> 140193037260832
140193037261552 [label=ViewBackward0]
140193037262704 -> 140193037261552
140193037262704 [label=AddmmBackward0]
140193037258864 -> 140193037262704
140193037258864 [label=ToCopyBackward0]
140193036049952 -> 140193037258864
140193039536752 [label="encoder.layer.4.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039536752 -> 140193036049952
140193036049952 [label=AccumulateGrad]
140193036050288 -> 140193037262704
140193036050288 [label=ViewBackward0]
140193036152992 -> 140193036050288
140193036152992 [label=ToCopyBackward0]
140193037276496 -> 140193036152992
140193036049904 -> 140193037262704
140193036049904 [label=TBackward0]
140193036152896 -> 140193036049904
140193036152896 [label=ToCopyBackward0]
140193036153136 -> 140193036152896
140193039537072 [label="encoder.layer.4.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039537072 -> 140193036153136
140193036153136 [label=AccumulateGrad]
140193037277552 -> 140193037277504
140193037277552 [label=UnsafeViewBackward0]
140193037277888 -> 140193037277552
140193037277888 [label=CloneBackward0]
140193037278080 -> 140193037277888
140193037278080 [label=ExpandBackward0]
140193037278272 -> 140193037278080
140193037278272 [label=PermuteBackward0]
140193037277648 -> 140193037278272
140193037277648 [label=ViewBackward0]
140193037278896 -> 140193037277648
140193037278896 [label=ViewBackward0]
140193037278608 -> 140193037278896
140193037278608 [label=AddmmBackward0]
140193037259104 -> 140193037278608
140193037259104 [label=ToCopyBackward0]
140193036050192 -> 140193037259104
140193039536512 [label="encoder.layer.4.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039536512 -> 140193036050192
140193036050192 [label=AccumulateGrad]
140193037261024 -> 140193037278608
140193037261024 [label=ViewBackward0]
140193036153232 -> 140193037261024
140193036153232 [label=ToCopyBackward0]
140193037276496 -> 140193036153232
140193037259152 -> 140193037278608
140193037259152 [label=TBackward0]
140193036153088 -> 140193037259152
140193036153088 [label=ToCopyBackward0]
140193036153280 -> 140193036153088
140193039536832 [label="encoder.layer.4.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039536832 -> 140193036153280
140193036153280 [label=AccumulateGrad]
140193037276592 -> 140193037276784
140193037276592 [label=TBackward0]
140193037277264 -> 140193037276592
140193037277264 [label=ToCopyBackward0]
140193037277456 -> 140193037277264
140193039536592 [label="encoder.layer.4.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039536592 -> 140193037277456
140193037277456 [label=AccumulateGrad]
140193037276496 -> 140193037276352
140193037276304 -> 140193037276256
140193039536352 [label="encoder.layer.4.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039536352 -> 140193037276304
140193037276304 [label=AccumulateGrad]
140193037275776 -> 140193037276256
140193039519552 [label="encoder.layer.4.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039519552 -> 140193037275776
140193037275776 [label=AccumulateGrad]
140193037275200 -> 140193037275584
140193037275200 [label=TBackward0]
140193037275824 -> 140193037275200
140193037275824 [label=ToCopyBackward0]
140193037276208 -> 140193037275824
140193039519632 [label="encoder.layer.4.crossattention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039519632 -> 140193037276208
140193037276208 [label=AccumulateGrad]
140193037364960 -> 140193037364912
140193037364960 [label=UnsafeViewBackward0]
140193037365056 -> 140193037364960
140193037365056 [label=CloneBackward0]
140193037275440 -> 140193037365056
140193037275440 [label=ExpandBackward0]
140193037275728 -> 140193037275440
140193037275728 [label=TransposeBackward0]
140193037276112 -> 140193037275728
140193037276112 [label=PermuteBackward0]
140193037276400 -> 140193037276112
140193037276400 [label=ViewBackward0]
140193037276640 -> 140193037276400
140193037276640 [label=ViewBackward0]
140193037276928 -> 140193037276640
140193037276928 [label=AddmmBackward0]
140193037277360 -> 140193037276928
140193037277360 [label=ToCopyBackward0]
140193037277984 -> 140193037277360
140193039519072 [label="encoder.layer.4.crossattention.self.key.bias
(768)" fillcolor=lightblue]
140193039519072 -> 140193037277984
140193037277984 [label=AccumulateGrad]
140193037276976 -> 140193037276928
140193037276976 [label=ViewBackward0]
140193037278176 -> 140193037276976
140193037278176 [label=ToCopyBackward0]
140193036037232 -> 140193037278176
140193037275248 -> 140193037276928
140193037275248 [label=TBackward0]
140193037277024 -> 140193037275248
140193037277024 [label=ToCopyBackward0]
140193037277696 -> 140193037277024
140193039519392 [label="encoder.layer.4.crossattention.self.key.weight
(768, 1408)" fillcolor=lightblue]
140193039519392 -> 140193037277696
140193037277696 [label=AccumulateGrad]
140193037364048 -> 140193037364000
140193037364048 [label=UnsafeViewBackward0]
140193037364384 -> 140193037364048
140193037364384 [label=CloneBackward0]
140193037364576 -> 140193037364384
140193037364576 [label=ExpandBackward0]
140193037364768 -> 140193037364576
140193037364768 [label=PermuteBackward0]
140193037364144 -> 140193037364768
140193037364144 [label=ViewBackward0]
140193037261648 -> 140193037364144
140193037261648 [label=ViewBackward0]
140193037364192 -> 140193037261648
140193037364192 [label=AddmmBackward0]
140193037276448 -> 140193037364192
140193037276448 [label=ToCopyBackward0]
140193037278704 -> 140193037276448
140193039518832 [label="encoder.layer.4.crossattention.self.value.bias
(768)" fillcolor=lightblue]
140193039518832 -> 140193037278704
140193037278704 [label=AccumulateGrad]
140193037276016 -> 140193037364192
140193037276016 [label=ViewBackward0]
140193037277168 -> 140193037276016
140193037277168 [label=ToCopyBackward0]
140193036037232 -> 140193037277168
140193037275344 -> 140193037364192
140193037275344 [label=TBackward0]
140193037277792 -> 140193037275344
140193037277792 [label=ToCopyBackward0]
140193037278368 -> 140193037277792
140193039519152 [label="encoder.layer.4.crossattention.self.value.weight
(768, 1408)" fillcolor=lightblue]
140193039519152 -> 140193037278368
140193037278368 [label=AccumulateGrad]
140193037363088 -> 140193037363280
140193037363088 [label=TBackward0]
140193037363760 -> 140193037363088
140193037363760 [label=ToCopyBackward0]
140193037363952 -> 140193037363760
140193039518912 [label="encoder.layer.4.crossattention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039518912 -> 140193037363952
140193037363952 [label=AccumulateGrad]
140193037362992 -> 140193037362848
140193037362800 -> 140193037362752
140193039518672 [label="encoder.layer.4.crossattention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039518672 -> 140193037362800
140193037362800 [label=AccumulateGrad]
140193037362368 -> 140193037362752
140193039518352 [label="encoder.layer.4.crossattention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039518352 -> 140193037362368
140193037362368 [label=AccumulateGrad]
140193037361888 -> 140193037362176
140193037361888 [label=TBackward0]
140193037362416 -> 140193037361888
140193037362416 [label=ToCopyBackward0]
140193037362896 -> 140193037362416
140193039516912 [label="encoder.layer.4.experts.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039516912 -> 140193037362896
140193037362896 [label=AccumulateGrad]
140193037361456 -> 140193037361648
140193037361456 [label=TBackward0]
140193037362128 -> 140193037361456
140193037362128 [label=ToCopyBackward0]
140193037362608 -> 140193037362128
140193039516672 [label="encoder.layer.4.experts.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039516672 -> 140193037362608
140193037362608 [label=AccumulateGrad]
140193037361360 -> 140193037275088
140193037275040 -> 140193037274992
140193039516432 [label="encoder.layer.4.expert_ln.weight
(768)" fillcolor=lightblue]
140193039516432 -> 140193037275040
140193037275040 [label=AccumulateGrad]
140193037361216 -> 140193037274992
140193039516512 [label="encoder.layer.4.expert_ln.bias
(768)" fillcolor=lightblue]
140193039516512 -> 140193037361216
140193037361216 [label=AccumulateGrad]
140193037274752 -> 140193037272592
140193037274752 [label=NativeLayerNormBackward0]
140193037361504 -> 140193037274752
140193037361504 [label=AddBackward0]
140193037362320 -> 140193037361504
140193037362320 [label=NativeDropoutBackward0]
140193037362032 -> 140193037362320
140193037362032 [label=ViewBackward0]
140193037362560 -> 140193037362032
140193037362560 [label=AddmmBackward0]
140193037363424 -> 140193037362560
140193037363424 [label=ToCopyBackward0]
140193037363520 -> 140193037363424
140193039518192 [label="encoder.layer.4.output.dense.bias
(768)" fillcolor=lightblue]
140193039518192 -> 140193037363520
140193037363520 [label=AccumulateGrad]
140193037363232 -> 140193037362560
140193037363232 [label=ViewBackward0]
140193037363664 -> 140193037363232
140193037363664 [label=GeluBackward0]
140193037364672 -> 140193037363664
140193037364672 [label=ViewBackward0]
140193037365104 -> 140193037364672
140193037365104 [label=AddmmBackward0]
140193037364288 -> 140193037365104
140193037364288 [label=ToCopyBackward0]
140193036153376 -> 140193037364288
140193039518432 [label="encoder.layer.4.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039518432 -> 140193036153376
140193036153376 [label=AccumulateGrad]
140193037275968 -> 140193037365104
140193037275968 [label=ViewBackward0]
140193036153040 -> 140193037275968
140193036153040 [label=ToCopyBackward0]
140193037361840 -> 140193036153040
140193037361840 [label=SliceBackward0]
140193036153472 -> 140193037361840
140193036153472 [label=SliceBackward0]
140193036153568 -> 140193036153472
140193036153568 [label=SliceBackward0]
140193037276256 -> 140193036153568
140193037275536 -> 140193037365104
140193037275536 [label=TBackward0]
140193036153328 -> 140193037275536
140193036153328 [label=ToCopyBackward0]
140193036153664 -> 140193036153328
140194225614656 [label="encoder.layer.4.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140194225614656 -> 140193036153664
140193036153664 [label=AccumulateGrad]
140193037363136 -> 140193037362560
140193037363136 [label=TBackward0]
140193037364864 -> 140193037363136
140193037364864 [label=ToCopyBackward0]
140193037276736 -> 140193037364864
140193039518112 [label="encoder.layer.4.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039518112 -> 140193037276736
140193037276736 [label=AccumulateGrad]
140193037361840 -> 140193037361504
140193037361312 -> 140193037274752
140193039517872 [label="encoder.layer.4.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039517872 -> 140193037361312
140193037361312 [label=AccumulateGrad]
140193037361264 -> 140193037274752
140193039517952 [label="encoder.layer.4.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039517952 -> 140193037361264
140193037361264 [label=AccumulateGrad]
140193037273936 -> 140193037274512
140193037273936 [label=TBackward0]
140193037274704 -> 140193037273936
140193037274704 [label=ToCopyBackward0]
140193037274896 -> 140193037274704
140193039516192 [label="encoder.layer.5.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039516192 -> 140193037274896
140193037274896 [label=AccumulateGrad]
140193037273840 -> 140193037273792
140193037273840 [label=UnsafeViewBackward0]
140193037274176 -> 140193037273840
140193037274176 [label=CloneBackward0]
140193037274368 -> 140193037274176
140193037274368 [label=ExpandBackward0]
140193037274656 -> 140193037274368
140193037274656 [label=TransposeBackward0]
140193037274944 -> 140193037274656
140193037274944 [label=PermuteBackward0]
140193037362704 -> 140193037274944
140193037362704 [label=ViewBackward0]
140193037363472 -> 140193037362704
140193037363472 [label=ViewBackward0]
140193037364480 -> 140193037363472
140193037364480 [label=AddmmBackward0]
140193037361792 -> 140193037364480
140193037361792 [label=ToCopyBackward0]
140193036153712 -> 140193037361792
140193039516032 [label="encoder.layer.5.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039516032 -> 140193036153712
140193036153712 [label=AccumulateGrad]
140193036153520 -> 140193037364480
140193036153520 [label=ViewBackward0]
140193036153760 -> 140193036153520
140193036153760 [label=ToCopyBackward0]
140193037272592 -> 140193036153760
140193036152944 -> 140193037364480
140193036152944 [label=TBackward0]
140193036153616 -> 140193036152944
140193036153616 [label=ToCopyBackward0]
140193036153904 -> 140193036153616
140193039515952 [label="encoder.layer.5.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039515952 -> 140193036153904
140193036153904 [label=AccumulateGrad]
140193037271536 -> 140193037271488
140193037271536 [label=UnsafeViewBackward0]
140193037271104 -> 140193037271536
140193037271104 [label=CloneBackward0]
140193037273456 -> 140193037271104
140193037273456 [label=ExpandBackward0]
140193037273648 -> 140193037273456
140193037273648 [label=PermuteBackward0]
140193037271440 -> 140193037273648
140193037271440 [label=ViewBackward0]
140193037274272 -> 140193037271440
140193037274272 [label=ViewBackward0]
140193037273984 -> 140193037274272
140193037273984 [label=AddmmBackward0]
140193037271296 -> 140193037273984
140193037271296 [label=ToCopyBackward0]
140193036153184 -> 140193037271296
140193039515792 [label="encoder.layer.5.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039515792 -> 140193036153184
140193036153184 [label=AccumulateGrad]
140193037361600 -> 140193037273984
140193037361600 [label=ViewBackward0]
140193036154000 -> 140193037361600
140193036154000 [label=ToCopyBackward0]
140193037272592 -> 140193036154000
140193037362944 -> 140193037273984
140193037362944 [label=TBackward0]
140193036153424 -> 140193037362944
140193036153424 [label=ToCopyBackward0]
140193036154048 -> 140193036153424
140193039515712 [label="encoder.layer.5.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039515712 -> 140193036154048
140193036154048 [label=AccumulateGrad]
140193037272496 -> 140193037272304
140193037272496 [label=TBackward0]
140193037271824 -> 140193037272496
140193037271824 [label=ToCopyBackward0]
140193037271632 -> 140193037271824
140193039498992 [label="encoder.layer.5.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039498992 -> 140193037271632
140193037271632 [label=AccumulateGrad]
140193037272592 -> 140193037272640
140193037272784 -> 140193037272736
140193039498752 [label="encoder.layer.5.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039498752 -> 140193037272784
140193037272784 [label=AccumulateGrad]
140193037273264 -> 140193037272736
140193039498832 [label="encoder.layer.5.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039498832 -> 140193037273264
140193037273264 [label=AccumulateGrad]
140193036303952 -> 140193036302896
140193036303952 [label=TBackward0]
140193036303472 -> 140193036303952
140193036303472 [label=ToCopyBackward0]
140193037273072 -> 140193036303472
140193039497072 [label="encoder.layer.5.experts.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039497072 -> 140193037273072
140193037273072 [label=AccumulateGrad]
140193036301744 -> 140193036302032
140193036301744 [label=TBackward0]
140193036304336 -> 140193036301744
140193036304336 [label=ToCopyBackward0]
140193036302704 -> 140193036304336
140193039496832 [label="encoder.layer.5.experts.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039496832 -> 140193036302704
140193036302704 [label=AccumulateGrad]
140193036301552 -> 140193036301456
140193036301264 -> 140193036301072
140193039496592 [label="encoder.layer.5.expert_ln.weight
(768)" fillcolor=lightblue]
140193039496592 -> 140193036301264
140193036301264 [label=AccumulateGrad]
140193036301360 -> 140193036301072
140193039496672 [label="encoder.layer.5.expert_ln.bias
(768)" fillcolor=lightblue]
140193039496672 -> 140193036301360
140193036301360 [label=AccumulateGrad]
140193036300880 -> 140193036271824
140193036300880 [label=NativeLayerNormBackward0]
140193036301840 -> 140193036300880
140193036301840 [label=AddBackward0]
140193036303088 -> 140193036301840
140193036303088 [label=NativeDropoutBackward0]
140193036302752 -> 140193036303088
140193036302752 [label=ViewBackward0]
140193037273216 -> 140193036302752
140193037273216 [label=AddmmBackward0]
140193037272544 -> 140193037273216
140193037272544 [label=ToCopyBackward0]
140193037272112 -> 140193037272544
140193039498352 [label="encoder.layer.5.output.dense.bias
(768)" fillcolor=lightblue]
140193039498352 -> 140193037272112
140193037272112 [label=AccumulateGrad]
140193037272688 -> 140193037273216
140193037272688 [label=ViewBackward0]
140193037271728 -> 140193037272688
140193037271728 [label=GeluBackward0]
140193037271968 -> 140193037271728
140193037271968 [label=ViewBackward0]
140193037273360 -> 140193037271968
140193037273360 [label=AddmmBackward0]
140193037273744 -> 140193037273360
140193037273744 [label=ToCopyBackward0]
140193037363856 -> 140193037273744
140193039498592 [label="encoder.layer.5.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039498592 -> 140193037363856
140193037363856 [label=AccumulateGrad]
140193037273552 -> 140193037273360
140193037273552 [label=ViewBackward0]
140193037274464 -> 140193037273552
140193037274464 [label=ToCopyBackward0]
140193036304192 -> 140193037274464
140193036304192 [label=SliceBackward0]
140193036153952 -> 140193036304192
140193036153952 [label=SliceBackward0]
140193036154240 -> 140193036153952
140193036154240 [label=SliceBackward0]
140193037272736 -> 140193036154240
140193037272064 -> 140193037273360
140193037272064 [label=TBackward0]
140193036154144 -> 140193037272064
140193036154144 [label=ToCopyBackward0]
140193036154336 -> 140193036154144
140193039498512 [label="encoder.layer.5.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039498512 -> 140193036154336
140193036154336 [label=AccumulateGrad]
140193037272880 -> 140193037273216
140193037272880 [label=TBackward0]
140193037271920 -> 140193037272880
140193037271920 [label=ToCopyBackward0]
140193037274080 -> 140193037271920
140193039498272 [label="encoder.layer.5.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039498272 -> 140193037274080
140193037274080 [label=AccumulateGrad]
140193036304192 -> 140193036301840
140193036302800 -> 140193036300880
140193039498032 [label="encoder.layer.5.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039498032 -> 140193036302800
140193036302800 [label=AccumulateGrad]
140193036301312 -> 140193036300880
140193039498112 [label="encoder.layer.5.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039498112 -> 140193036301312
140193036301312 [label=AccumulateGrad]
140193036304048 -> 140193036300400
140193036304048 [label=TBackward0]
140193036300592 -> 140193036304048
140193036300592 [label=ToCopyBackward0]
140193036302320 -> 140193036300592
140193039496352 [label="encoder.layer.6.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039496352 -> 140193036302320
140193036302320 [label=AccumulateGrad]
140193036275184 -> 140193036275280
140193036275184 [label=UnsafeViewBackward0]
140193036275568 -> 140193036275184
140193036275568 [label=CloneBackward0]
140193036302512 -> 140193036275568
140193036302512 [label=ExpandBackward0]
140193036300688 -> 140193036302512
140193036300688 [label=TransposeBackward0]
140193036301936 -> 140193036300688
140193036301936 [label=PermuteBackward0]
140193036300832 -> 140193036301936
140193036300832 [label=ViewBackward0]
140193037272352 -> 140193036300832
140193037272352 [label=ViewBackward0]
140193037271200 -> 140193037272352
140193037271200 [label=AddmmBackward0]
140193037272976 -> 140193037271200
140193037272976 [label=ToCopyBackward0]
140193036154384 -> 140193037272976
140193039496192 [label="encoder.layer.6.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039496192 -> 140193036154384
140193036154384 [label=AccumulateGrad]
140193036154192 -> 140193037271200
140193036154192 [label=ViewBackward0]
140193036154432 -> 140193036154192
140193036154432 [label=ToCopyBackward0]
140193036271824 -> 140193036154432
140193036153808 -> 140193037271200
140193036153808 [label=TBackward0]
140193036154288 -> 140193036153808
140193036154288 [label=ToCopyBackward0]
140193036154576 -> 140193036154288
140193039496112 [label="encoder.layer.6.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039496112 -> 140193036154576
140193036154576 [label=AccumulateGrad]
140193036273552 -> 140193036273648
140193036273552 [label=UnsafeViewBackward0]
140193036274320 -> 140193036273552
140193036274320 [label=CloneBackward0]
140193036274512 -> 140193036274320
140193036274512 [label=ExpandBackward0]
140193036274752 -> 140193036274512
140193036274752 [label=PermuteBackward0]
140193036274416 -> 140193036274752
140193036274416 [label=ViewBackward0]
140193036273936 -> 140193036274416
140193036273936 [label=ViewBackward0]
140193036301168 -> 140193036273936
140193036301168 [label=AddmmBackward0]
140193036303376 -> 140193036301168
140193036303376 [label=ToCopyBackward0]
140193036153856 -> 140193036303376
140193039495952 [label="encoder.layer.6.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039495952 -> 140193036153856
140193036153856 [label=AccumulateGrad]
140193036303856 -> 140193036301168
140193036303856 [label=ViewBackward0]
140193036154672 -> 140193036303856
140193036154672 [label=ToCopyBackward0]
140193036271824 -> 140193036154672
140193037273168 -> 140193036301168
140193037273168 [label=TBackward0]
140193036154096 -> 140193037273168
140193036154096 [label=ToCopyBackward0]
140193036154720 -> 140193036154096
140193039495872 [label="encoder.layer.6.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039495872 -> 140193036154720
140193036154720 [label=AccumulateGrad]
140193036271872 -> 140193036272304
140193036271872 [label=TBackward0]
140193036273072 -> 140193036271872
140193036273072 [label=ToCopyBackward0]
140193036273312 -> 140193036273072
140193039495632 [label="encoder.layer.6.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039495632 -> 140193036273312
140193036273312 [label=AccumulateGrad]
140193036271824 -> 140193036250848
140193036250224 -> 140193036250992
140193039495392 [label="encoder.layer.6.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039495392 -> 140193036250224
140193036250224 [label=AccumulateGrad]
140193036271680 -> 140193036250992
140193039495472 [label="encoder.layer.6.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039495472 -> 140193036271680
140193036271680 [label=AccumulateGrad]
140193036248976 -> 140193036249936
140193036248976 [label=TBackward0]
140193036250128 -> 140193036248976
140193036250128 [label=ToCopyBackward0]
140193036250800 -> 140193036250128
140193039495232 [label="encoder.layer.6.crossattention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039495232 -> 140193036250800
140193036250800 [label=AccumulateGrad]
140193036248784 -> 140193036248448
140193036248784 [label=UnsafeViewBackward0]
140193036249168 -> 140193036248784
140193036249168 [label=CloneBackward0]
140193036249408 -> 140193036249168
140193036249408 [label=ExpandBackward0]
140193036249888 -> 140193036249408
140193036249888 [label=TransposeBackward0]
140193036250608 -> 140193036249888
140193036250608 [label=PermuteBackward0]
140193036250512 -> 140193036250608
140193036250512 [label=ViewBackward0]
140193036272208 -> 140193036250512
140193036272208 [label=ViewBackward0]
140193036272688 -> 140193036272208
140193036272688 [label=AddmmBackward0]
140193036273264 -> 140193036272688
140193036273264 [label=ToCopyBackward0]
140193036274272 -> 140193036273264
140193039490800 [label="encoder.layer.6.crossattention.self.key.bias
(768)" fillcolor=lightblue]
140193039490800 -> 140193036274272
140193036274272 [label=AccumulateGrad]
140193036272592 -> 140193036272688
140193036272592 [label=ViewBackward0]
140193036274704 -> 140193036272592
140193036274704 [label=ToCopyBackward0]
140193036037232 -> 140193036274704
140193036271920 -> 140193036272688
140193036271920 [label=TBackward0]
140193036272832 -> 140193036271920
140193036272832 [label=ToCopyBackward0]
140193036300352 -> 140193036272832
140193039490720 [label="encoder.layer.6.crossattention.self.key.weight
(768, 1408)" fillcolor=lightblue]
140193039490720 -> 140193036300352
140193036300352 [label=AccumulateGrad]
140193036247152 -> 140193036214128
140193036247152 [label=UnsafeViewBackward0]
140193036247824 -> 140193036247152
140193036247824 [label=CloneBackward0]
140193036248112 -> 140193036247824
140193036248112 [label=ExpandBackward0]
140193036248496 -> 140193036248112
140193036248496 [label=PermuteBackward0]
140193036247248 -> 140193036248496
140193036247248 [label=ViewBackward0]
140193036249360 -> 140193036247248
140193036249360 [label=ViewBackward0]
140193036250368 -> 140193036249360
140193036250368 [label=AddmmBackward0]
140193036303232 -> 140193036250368
140193036303232 [label=ToCopyBackward0]
140193036274128 -> 140193036303232
140193039490560 [label="encoder.layer.6.crossattention.self.value.bias
(768)" fillcolor=lightblue]
140193039490560 -> 140193036274128
140193036274128 [label=AccumulateGrad]
140193036248880 -> 140193036250368
140193036248880 [label=ViewBackward0]
140193036272880 -> 140193036248880
140193036272880 [label=ToCopyBackward0]
140193036037232 -> 140193036272880
140193036247536 -> 140193036250368
140193036247536 [label=TBackward0]
140193036272400 -> 140193036247536
140193036272400 [label=ToCopyBackward0]
140193036274992 -> 140193036272400
140193039490480 [label="encoder.layer.6.crossattention.self.value.weight
(768, 1408)" fillcolor=lightblue]
140193039490480 -> 140193036274992
140193036274992 [label=AccumulateGrad]
140193036212688 -> 140193036212976
140193036212688 [label=TBackward0]
140193036213696 -> 140193036212688
140193036213696 [label=ToCopyBackward0]
140193036213552 -> 140193036213696
140193039490240 [label="encoder.layer.6.crossattention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039490240 -> 140193036213552
140193036213552 [label=AccumulateGrad]
140193036212496 -> 140193036212400
140193036212208 -> 140193036212304
140193039490000 [label="encoder.layer.6.crossattention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039490000 -> 140193036212208
140193036212208 [label=AccumulateGrad]
140193036211632 -> 140193036212304
140193039490080 [label="encoder.layer.6.crossattention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039490080 -> 140193036211632
140193036211632 [label=AccumulateGrad]
140193036210576 -> 140193036211056
140193036210576 [label=TBackward0]
140193036211776 -> 140193036210576
140193036211776 [label=ToCopyBackward0]
140193036212256 -> 140193036211776
140193039487120 [label="encoder.layer.6.experts.experts.0.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039487120 -> 140193036212256
140193036212256 [label=AccumulateGrad]
140193036210384 -> 140193036713840
140193036210384 [label=TBackward0]
140193036211152 -> 140193036210384
140193036211152 [label=ToCopyBackward0]
140193036211296 -> 140193036211152
140193039487440 [label="encoder.layer.6.experts.experts.0.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039487440 -> 140193036211296
140193036211296 [label=AccumulateGrad]
140193036713456 -> 140193036713120
140193036713456 [label=UnsqueezeBackward0]
140193036713648 -> 140193036713456
140193036713648 [label=NativeDropoutBackward0]
140193036713552 -> 140193036713648
140193036713552 [label=ViewBackward0]
140193036212880 -> 140193036713552
140193036212880 [label=AddmmBackward0]
140193036210960 -> 140193036212880
140193036210960 [label=ToCopyBackward0]
140193036213360 -> 140193036210960
140193039469680 [label="encoder.layer.6.experts.experts.1.dense2.bias
(768)" fillcolor=lightblue]
140193039469680 -> 140193036213360
140193036213360 [label=AccumulateGrad]
140193036212016 -> 140193036212880
140193036212016 [label=ViewBackward0]
140193036213216 -> 140193036212016
140193036213216 [label=GeluBackward0]
140193036213648 -> 140193036213216
140193036213648 [label=ViewBackward0]
140193036213072 -> 140193036213648
140193036213072 [label=AddmmBackward0]
140193036248304 -> 140193036213072
140193036248304 [label=ToCopyBackward0]
140193037272256 -> 140193036248304
140193039469920 [label="encoder.layer.6.experts.experts.1.dense1.bias
(3072)" fillcolor=lightblue]
140193039469920 -> 140193037272256
140193037272256 [label=AccumulateGrad]
140193036248016 -> 140193036213072
140193036248016 [label=ViewBackward0]
140193036248928 -> 140193036248016
140193036248928 [label=ToCopyBackward0]
140193036211920 -> 140193036248928
140193036247632 -> 140193036213072
140193036247632 [label=TBackward0]
140193036271728 -> 140193036247632
140193036271728 [label=ToCopyBackward0]
140193036154816 -> 140193036271728
140193039470240 [label="encoder.layer.6.experts.experts.1.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039470240 -> 140193036154816
140193036154816 [label=AccumulateGrad]
140193036210336 -> 140193036212880
140193036210336 [label=TBackward0]
140193036211728 -> 140193036210336
140193036211728 [label=ToCopyBackward0]
140193036249648 -> 140193036211728
140193039470000 [label="encoder.layer.6.experts.experts.1.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039470000 -> 140193036249648
140193036249648 [label=AccumulateGrad]
140193036712112 -> 140193036712208
140193036712112 [label=UnsqueezeBackward0]
140193036712784 -> 140193036712112
140193036712784 [label=UnsqueezeBackward0]
140193036713168 -> 140193036712784
140193036713168 [label=MulBackward0]
140193036712160 -> 140193036713168
140193036712160 [label=IndexBackward0]
140193036713600 -> 140193036712160
140193036713600 [label=ViewBackward0]
140193036212592 -> 140193036713600
140193036212592 [label=CloneBackward0]
140193036213936 -> 140193036212592
140193036213936 [label=ExpandBackward0]
140193036247104 -> 140193036213936
140193036247104 [label=UnsqueezeBackward0]
140193036154480 -> 140193036247104
140193036154480 [label=SoftmaxBackward0]
140193036154864 -> 140193036154480
140193036154864 [label=CatBackward0]
140193036154960 -> 140193036154864
140193036154960 [label=MmBackward0]
140193036155104 -> 140193036154960
140193036155104 [label=MeanBackward1]
140193036713744 -> 140193036155104
140193036155056 -> 140193036154960
140193036155056 [label=TBackward0]
140193036155152 -> 140193036155056
140193036155152 [label=ToCopyBackward0]
140193036155344 -> 140193036155152
140193039487840 [label="encoder.layer.6.experts.gate.weight
(1, 768)" fillcolor=lightblue]
140193039487840 -> 140193036155344
140193036155344 [label=AccumulateGrad]
140193036154912 -> 140193036154864
140193036154912 [label=MmBackward0]
140193036155296 -> 140193036154912
140193036155296 [label=MeanBackward1]
140193036713648 -> 140193036155296
140193036155200 -> 140193036154912
140193036155200 [label=TBackward0]
140193036155152 -> 140193036155200
140193036711344 -> 140193036711056
140193036711344 [label=UnsqueezeBackward0]
140193036711680 -> 140193036711344
140193036711680 [label=SelectBackward0]
140193036711536 -> 140193036711680
140193036711536 [label=NativeDropoutBackward0]
140193036712976 -> 140193036711536
140193036712976 [label=ViewBackward0]
140193036712496 -> 140193036712976
140193036712496 [label=AddmmBackward0]
140193036210672 -> 140193036712496
140193036210672 [label=ToCopyBackward0]
140193036154624 -> 140193036210672
140193039469200 [label="encoder.layer.6.experts.experts.2.dense2.bias
(768)" fillcolor=lightblue]
140193039469200 -> 140193036154624
140193036154624 [label=AccumulateGrad]
140193036211440 -> 140193036712496
140193036211440 [label=ViewBackward0]
140193036155008 -> 140193036211440
140193036155008 [label=GeluBackward0]
140193036155392 -> 140193036155008
140193036155392 [label=ViewBackward0]
140193036155488 -> 140193036155392
140193036155488 [label=AddmmBackward0]
140193036155584 -> 140193036155488
140193036155584 [label=ToCopyBackward0]
140193036155776 -> 140193036155584
140193039469440 [label="encoder.layer.6.experts.experts.2.dense1.bias
(3072)" fillcolor=lightblue]
140193039469440 -> 140193036155776
140193036155776 [label=AccumulateGrad]
140193036155536 -> 140193036155488
140193036155536 [label=ViewBackward0]
140193036155824 -> 140193036155536
140193036155824 [label=ToCopyBackward0]
140193036211920 -> 140193036155824
140193036154768 -> 140193036155488
140193036154768 [label=TBackward0]
140193036155680 -> 140193036154768
140193036155680 [label=ToCopyBackward0]
140193036155968 -> 140193036155680
140193039469760 [label="encoder.layer.6.experts.experts.2.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039469760 -> 140193036155968
140193036155968 [label=AccumulateGrad]
140193036711440 -> 140193036712496
140193036711440 [label=TBackward0]
140193036155248 -> 140193036711440
140193036155248 [label=ToCopyBackward0]
140193036155920 -> 140193036155248
140193039469520 [label="encoder.layer.6.experts.experts.2.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039469520 -> 140193036155920
140193036155920 [label=AccumulateGrad]
140193036710720 -> 140193036710672
140193036710720 [label=CatBackward0]
140193036711920 -> 140193036710720
140193036711920 [label=SliceBackward0]
140193036248592 -> 140193036711920
140193036248592 [label=SliceBackward0]
140193036712688 -> 140193036248592
140193036712688 [label=SliceBackward0]
140193036712016 -> 140193036712688
140193036711632 -> 140193036710720
140193036711632 [label=UnsqueezeBackward0]
140193036713264 -> 140193036711632
140193036713264 [label=SelectBackward0]
140193036711536 -> 140193036713264
140193036710864 -> 140193036710672
140193036710864 [label=CatBackward0]
140193036711152 -> 140193036710864
140193036711152 [label=SliceBackward0]
140193036155440 -> 140193036711152
140193036155440 [label=SliceBackward0]
140193036155872 -> 140193036155440
140193036155872 [label=SliceBackward0]
140193036712016 -> 140193036155872
140193036154528 -> 140193036710864
140193036154528 [label=UnsqueezeBackward0]
140193036156160 -> 140193036154528
140193036156160 [label=SelectBackward0]
140193036711536 -> 140193036156160
140193036710960 -> 140193036710672
140193036710960 [label=CatBackward0]
140193036156064 -> 140193036710960
140193036156064 [label=SliceBackward0]
140193036156208 -> 140193036156064
140193036156208 [label=SliceBackward0]
140193036156304 -> 140193036156208
140193036156304 [label=SliceBackward0]
140193036712016 -> 140193036156304
140193036156016 -> 140193036710960
140193036156016 [label=UnsqueezeBackward0]
140193036156400 -> 140193036156016
140193036156400 [label=SelectBackward0]
140193036711536 -> 140193036156400
140193036710768 -> 140193036710240
140193036710768 [label=ViewBackward0]
140193036711248 -> 140193036710768
140193036711248 [label=CloneBackward0]
140193036156352 -> 140193036711248
140193036156352 [label=ExpandBackward0]
140193036156448 -> 140193036156352
140193036156448 [label=UnsqueezeBackward0]
140193036211920 -> 140193036156448
140193036710384 -> 140193036710288
140194225780112 [label="encoder.layer.6.expert_ln.weight
(768)" fillcolor=lightblue]
140194225780112 -> 140193036710384
140193036710384 [label=AccumulateGrad]
140193036710192 -> 140193036710288
140193039487360 [label="encoder.layer.6.expert_ln.bias
(768)" fillcolor=lightblue]
140193039487360 -> 140193036710192
140193036710192 [label=AccumulateGrad]
140193036710096 -> 140193036659728
140193036710096 [label=ViewBackward0]
140193036710480 -> 140193036710096
140193036710480 [label=CloneBackward0]
140193036155632 -> 140193036710480
140193036155632 [label=ExpandBackward0]
140193036156496 -> 140193036155632
140193036156496 [label=UnsqueezeBackward0]
140193036156592 -> 140193036156496
140193036156592 [label=NativeLayerNormBackward0]
140193036156688 -> 140193036156592
140193036156688 [label=AddBackward0]
140193036156880 -> 140193036156688
140193036156880 [label=NativeDropoutBackward0]
140201394335904 -> 140193036156880
140201394335904 [label=ViewBackward0]
140201394336000 -> 140201394335904
140201394336000 [label=AddmmBackward0]
140201394336096 -> 140201394336000
140201394336096 [label=ToCopyBackward0]
140201394336288 -> 140201394336096
140193039489600 [label="encoder.layer.6.output.dense.bias
(768)" fillcolor=lightblue]
140193039489600 -> 140201394336288
140201394336288 [label=AccumulateGrad]
140201394336048 -> 140201394336000
140201394336048 [label=ViewBackward0]
140201394336336 -> 140201394336048
140201394336336 [label=GeluBackward0]
140201394336432 -> 140201394336336
140201394336432 [label=ViewBackward0]
140201394336528 -> 140201394336432
140201394336528 [label=AddmmBackward0]
140201394336624 -> 140201394336528
140201394336624 [label=ToCopyBackward0]
140201394336816 -> 140201394336624
140193039489840 [label="encoder.layer.6.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039489840 -> 140201394336816
140201394336816 [label=AccumulateGrad]
140201394336576 -> 140201394336528
140201394336576 [label=ViewBackward0]
140201394336864 -> 140201394336576
140201394336864 [label=ToCopyBackward0]
140193036156832 -> 140201394336864
140193036156832 [label=SliceBackward0]
140201394337008 -> 140193036156832
140201394337008 [label=SliceBackward0]
140201394337104 -> 140201394337008
140201394337104 [label=SliceBackward0]
140193036250992 -> 140201394337104
140201394336240 -> 140201394336528
140201394336240 [label=TBackward0]
140201394336768 -> 140201394336240
140201394336768 [label=ToCopyBackward0]
140201394337200 -> 140201394336768
140193039489760 [label="encoder.layer.6.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039489760 -> 140201394337200
140201394337200 [label=AccumulateGrad]
140201394335808 -> 140201394336000
140201394335808 [label=TBackward0]
140201394336480 -> 140201394335808
140201394336480 [label=ToCopyBackward0]
140201394336960 -> 140201394336480
140193039489520 [label="encoder.layer.6.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039489520 -> 140201394336960
140201394336960 [label=AccumulateGrad]
140193036156832 -> 140193036156688
140193036156640 -> 140193036156592
140193039489280 [label="encoder.layer.6.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039489280 -> 140193036156640
140193036156640 [label=AccumulateGrad]
140193036156256 -> 140193036156592
140193039489360 [label="encoder.layer.6.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039489360 -> 140193036156256
140193036156256 [label=AccumulateGrad]
140193036679728 -> 140193036680688
140193036679728 [label=TBackward0]
140193036680976 -> 140193036679728
140193036680976 [label=ToCopyBackward0]
140193036710576 -> 140193036680976
140193039488160 [label="encoder.layer.7.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039488160 -> 140193036710576
140193036710576 [label=AccumulateGrad]
140193036679632 -> 140193036679344
140193036679632 [label=UnsafeViewBackward0]
140193036680016 -> 140193036679632
140193036680016 [label=CloneBackward0]
140193036680304 -> 140193036680016
140193036680304 [label=ExpandBackward0]
140193036680784 -> 140193036680304
140193036680784 [label=TransposeBackward0]
140193036710000 -> 140193036680784
140193036710000 [label=PermuteBackward0]
140193036679584 -> 140193036710000
140193036679584 [label=ViewBackward0]
140193036156736 -> 140193036679584
140193036156736 [label=ViewBackward0]
140193036156784 -> 140193036156736
140193036156784 [label=AddmmBackward0]
140201394336144 -> 140193036156784
140201394336144 [label=ToCopyBackward0]
140201394337056 -> 140201394336144
140193039488320 [label="encoder.layer.7.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039488320 -> 140201394337056
140201394337056 [label=AccumulateGrad]
140201394335952 -> 140193036156784
140201394335952 [label=ViewBackward0]
140201394336384 -> 140201394335952
140201394336384 [label=ToCopyBackward0]
140193036659728 -> 140201394336384
140201394335856 -> 140193036156784
140201394335856 [label=TBackward0]
140201394336672 -> 140201394335856
140201394336672 [label=ToCopyBackward0]
140201394337248 -> 140201394336672
140193039488400 [label="encoder.layer.7.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039488400 -> 140201394337248
140201394337248 [label=AccumulateGrad]
140193036677904 -> 140193036678000
140193036677904 [label=UnsafeViewBackward0]
140193036678672 -> 140193036677904
140193036678672 [label=CloneBackward0]
140193036678960 -> 140193036678672
140193036678960 [label=ExpandBackward0]
140193036679248 -> 140193036678960
140193036679248 [label=PermuteBackward0]
140193036678096 -> 140193036679248
140193036678096 [label=ViewBackward0]
140193036680064 -> 140193036678096
140193036680064 [label=ViewBackward0]
140193036681024 -> 140193036680064
140193036681024 [label=AddmmBackward0]
140193036678288 -> 140193036681024
140193036678288 [label=ToCopyBackward0]
140201394336720 -> 140193036678288
140193039487920 [label="encoder.layer.7.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039487920 -> 140201394336720
140201394336720 [label=AccumulateGrad]
140193036156112 -> 140193036681024
140193036156112 [label=ViewBackward0]
140201394337344 -> 140193036156112
140201394337344 [label=ToCopyBackward0]
140193036659728 -> 140201394337344
140193036156544 -> 140193036681024
140193036156544 [label=TBackward0]
140201394336192 -> 140193036156544
140201394336192 [label=ToCopyBackward0]
140201394337392 -> 140201394336192
140193039487600 [label="encoder.layer.7.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039487600 -> 140201394337392
140201394337392 [label=AccumulateGrad]
140193036659776 -> 140193036660208
140193036659776 [label=TBackward0]
140193036677424 -> 140193036659776
140193036677424 [label=ToCopyBackward0]
140193036677664 -> 140193036677424
140193039487680 [label="encoder.layer.7.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039487680 -> 140193036677664
140193036677664 [label=AccumulateGrad]
140193036659728 -> 140193036659632
140193036659296 -> 140193036659440
140193039468960 [label="encoder.layer.7.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039468960 -> 140193036659296
140193036659296 [label=AccumulateGrad]
140193036658672 -> 140193036659440
140193039470160 [label="encoder.layer.7.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039470160 -> 140193036658672
140193036658672 [label=AccumulateGrad]
140193036657424 -> 140193036657904
140193036657424 [label=TBackward0]
140193036658576 -> 140193036657424
140193036658576 [label=ToCopyBackward0]
140193036658768 -> 140193036658576
140193039461968 [label="encoder.layer.7.experts.experts.0.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039461968 -> 140193036658768
140193036658768 [label=AccumulateGrad]
140193036656704 -> 140193036656848
140193036656704 [label=TBackward0]
140193036657616 -> 140193036656704
140193036657616 [label=ToCopyBackward0]
140193036658192 -> 140193036657616
140193039461648 [label="encoder.layer.7.experts.experts.0.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039461648 -> 140193036658192
140193036658192 [label=AccumulateGrad]
140193036631536 -> 140193036631632
140193036631536 [label=UnsqueezeBackward0]
140193036631920 -> 140193036631536
140193036631920 [label=NativeDropoutBackward0]
140193036657136 -> 140193036631920
140193036657136 [label=ViewBackward0]
140193036659056 -> 140193036657136
140193036659056 [label=AddmmBackward0]
140193036657376 -> 140193036659056
140193036657376 [label=ToCopyBackward0]
140193036659536 -> 140193036657376
140193039461728 [label="encoder.layer.7.experts.experts.1.dense2.bias
(768)" fillcolor=lightblue]
140193039461728 -> 140193036659536
140193036659536 [label=AccumulateGrad]
140193036658096 -> 140193036659056
140193036658096 [label=ViewBackward0]
140193036659824 -> 140193036658096
140193036659824 [label=GeluBackward0]
140193036660304 -> 140193036659824
140193036660304 [label=ViewBackward0]
140193036660592 -> 140193036660304
140193036660592 [label=AddmmBackward0]
140193036678480 -> 140193036660592
140193036678480 [label=ToCopyBackward0]
140193036679152 -> 140193036678480
140193039462128 [label="encoder.layer.7.experts.experts.1.dense1.bias
(3072)" fillcolor=lightblue]
140193039462128 -> 140193036679152
140193036679152 [label=AccumulateGrad]
140193036677616 -> 140193036660592
140193036677616 [label=ViewBackward0]
140193036679440 -> 140193036677616
140193036679440 [label=ToCopyBackward0]
140193036658336 -> 140193036679440
140193036677184 -> 140193036660592
140193036677184 [label=TBackward0]
140193036677328 -> 140193036677184
140193036677328 [label=ToCopyBackward0]
140193036155728 -> 140193036677328
140193039461408 [label="encoder.layer.7.experts.experts.1.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039461408 -> 140193036155728
140193036155728 [label=AccumulateGrad]
140193036656752 -> 140193036659056
140193036656752 [label=TBackward0]
140193036659248 -> 140193036656752
140193036659248 [label=ToCopyBackward0]
140193036660112 -> 140193036659248
140193039461168 [label="encoder.layer.7.experts.experts.1.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039461168 -> 140193036660112
140193036660112 [label=AccumulateGrad]
140193036631152 -> 140193036630816
140193036631152 [label=UnsqueezeBackward0]
140193036631344 -> 140193036631152
140193036631344 [label=UnsqueezeBackward0]
140193036631248 -> 140193036631344
140193036631248 [label=MulBackward0]
140193036658816 -> 140193036631248
140193036658816 [label=IndexBackward0]
140193036658480 -> 140193036658816
140193036658480 [label=SoftmaxBackward0]
140193036678768 -> 140193036658480
140193036678768 [label=CatBackward0]
140193036677232 -> 140193036678768
140193036677232 [label=MmBackward0]
140201394337440 -> 140193036677232
140201394337440 [label=MeanBackward1]
140193036631776 -> 140201394337440
140201394337488 -> 140193036677232
140201394337488 [label=TBackward0]
140201394337296 -> 140201394337488
140201394337296 [label=ToCopyBackward0]
140201394337680 -> 140201394337296
140193039467120 [label="encoder.layer.7.experts.gate.weight
(1, 768)" fillcolor=lightblue]
140193039467120 -> 140201394337680
140201394337680 [label=AccumulateGrad]
140193036680496 -> 140193036678768
140193036680496 [label=MmBackward0]
140201394337632 -> 140193036680496
140201394337632 [label=MeanBackward1]
140193036631920 -> 140201394337632
140201394337536 -> 140193036680496
140201394337536 [label=TBackward0]
140201394337296 -> 140201394337536
140193036630096 -> 140193036629808
140193036630096 [label=UnsqueezeBackward0]
140193036630864 -> 140193036630096
140193036630864 [label=SelectBackward0]
140193036630288 -> 140193036630864
140193036630288 [label=NativeDropoutBackward0]
140193036631296 -> 140193036630288
140193036631296 [label=ViewBackward0]
140193036656944 -> 140193036631296
140193036656944 [label=AddmmBackward0]
140193036679824 -> 140193036656944
140193036679824 [label=ToCopyBackward0]
140201394337584 -> 140193036679824
140193039461248 [label="encoder.layer.7.experts.experts.2.dense2.bias
(768)" fillcolor=lightblue]
140193039461248 -> 140201394337584
140201394337584 [label=AccumulateGrad]
140193036657232 -> 140193036656944
140193036657232 [label=ViewBackward0]
140201394337824 -> 140193036657232
140201394337824 [label=GeluBackward0]
140201394337920 -> 140201394337824
140201394337920 [label=ViewBackward0]
140201394338016 -> 140201394337920
140201394338016 [label=AddmmBackward0]
140201394338112 -> 140201394338016
140201394338112 [label=ToCopyBackward0]
140201394338304 -> 140201394338112
140193039461488 [label="encoder.layer.7.experts.experts.2.dense1.bias
(3072)" fillcolor=lightblue]
140193039461488 -> 140201394338304
140201394338304 [label=AccumulateGrad]
140201394338064 -> 140201394338016
140201394338064 [label=ViewBackward0]
140201394338352 -> 140201394338064
140201394338352 [label=ToCopyBackward0]
140201394338448 -> 140201394338352
140201394338448 [label=IndexBackward0]
140193036629376 -> 140201394338448
140201394337728 -> 140201394338016
140201394337728 [label=TBackward0]
140201394338544 -> 140201394337728
140201394338544 [label=ToCopyBackward0]
140201394338256 -> 140201394338544
140193039460928 [label="encoder.layer.7.experts.experts.2.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039460928 -> 140201394338256
140201394338256 [label=AccumulateGrad]
140201394336912 -> 140193036656944
140201394336912 [label=TBackward0]
140201394337968 -> 140201394336912
140201394337968 [label=ToCopyBackward0]
140201394338208 -> 140201394337968
140193039460688 [label="encoder.layer.7.experts.experts.2.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039460688 -> 140201394338208
140201394338208 [label=AccumulateGrad]
140193036629904 -> 140193036629712
140193036629904 [label=CatBackward0]
140193036630960 -> 140193036629904
140193036630960 [label=SliceBackward0]
140193036657856 -> 140193036630960
140193036657856 [label=SliceBackward0]
140201394338160 -> 140193036657856
140201394338160 [label=SliceBackward0]
140193036630768 -> 140201394338160
140193036630672 -> 140193036629904
140193036630672 [label=UnsqueezeBackward0]
140193036630480 -> 140193036630672
140193036630480 [label=SelectBackward0]
140193036630288 -> 140193036630480
140193036629616 -> 140193036629712
140193036629616 [label=CatBackward0]
140193036630192 -> 140193036629616
140193036630192 [label=SliceBackward0]
140201394337872 -> 140193036630192
140201394337872 [label=SliceBackward0]
140201394338400 -> 140201394337872
140201394338400 [label=SliceBackward0]
140193036630768 -> 140201394338400
140201394337152 -> 140193036629616
140201394337152 [label=UnsqueezeBackward0]
140201394338736 -> 140201394337152
140201394338736 [label=SelectBackward0]
140193036630288 -> 140201394338736
140193036630000 -> 140193036629712
140193036630000 [label=CatBackward0]
140201394338640 -> 140193036630000
140201394338640 [label=SliceBackward0]
140201394338784 -> 140201394338640
140201394338784 [label=SliceBackward0]
140201394338880 -> 140201394338784
140201394338880 [label=SliceBackward0]
140193036630768 -> 140201394338880
140201394338592 -> 140193036630000
140201394338592 [label=UnsqueezeBackward0]
140201394338976 -> 140201394338592
140201394338976 [label=SelectBackward0]
140193036630288 -> 140201394338976
140193036629376 -> 140193036629424
140193036629136 -> 140193036628896
140193039467040 [label="encoder.layer.7.expert_ln.weight
(768)" fillcolor=lightblue]
140193039467040 -> 140193036629136
140193036629136 [label=AccumulateGrad]
140193036629232 -> 140193036628896
140193039466800 [label="encoder.layer.7.expert_ln.bias
(768)" fillcolor=lightblue]
140193039466800 -> 140193036629232
140193036629232 [label=AccumulateGrad]
140193036628752 -> 140193036570576
140193036628752 [label=IndexBackward0]
140193036629856 -> 140193036628752
140193036629856 [label=NativeLayerNormBackward0]
140193036629328 -> 140193036629856
140193036629328 [label=AddBackward0]
140201394339024 -> 140193036629328
140201394339024 [label=NativeDropoutBackward0]
140201394339168 -> 140201394339024
140201394339168 [label=ViewBackward0]
140201394339264 -> 140201394339168
140201394339264 [label=AddmmBackward0]
140201394339360 -> 140201394339264
140201394339360 [label=ToCopyBackward0]
140201394339552 -> 140201394339360
140193039468800 [label="encoder.layer.7.output.dense.bias
(768)" fillcolor=lightblue]
140193039468800 -> 140201394339552
140201394339552 [label=AccumulateGrad]
140201394339312 -> 140201394339264
140201394339312 [label=ViewBackward0]
140201394339600 -> 140201394339312
140201394339600 [label=GeluBackward0]
140201394339696 -> 140201394339600
140201394339696 [label=ViewBackward0]
140201394339792 -> 140201394339696
140201394339792 [label=AddmmBackward0]
140201394339504 -> 140201394339792
140201394339504 [label=ToCopyBackward0]
140201394377008 -> 140201394339504
140193039469040 [label="encoder.layer.7.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039469040 -> 140201394377008
140201394377008 [label=AccumulateGrad]
140201394376816 -> 140201394339792
140201394376816 [label=ViewBackward0]
140201394377056 -> 140201394376816
140201394377056 [label=ToCopyBackward0]
140201394337776 -> 140201394377056
140201394337776 [label=SliceBackward0]
140201394377200 -> 140201394337776
140201394377200 [label=SliceBackward0]
140201394377296 -> 140201394377200
140201394377296 [label=SliceBackward0]
140193036659440 -> 140201394377296
140201394376768 -> 140201394339792
140201394376768 [label=TBackward0]
140201394376960 -> 140201394376768
140201394376960 [label=ToCopyBackward0]
140201394377392 -> 140201394376960
140193039469280 [label="encoder.layer.7.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039469280 -> 140201394377392
140201394377392 [label=AccumulateGrad]
140201394339072 -> 140201394339264
140201394339072 [label=TBackward0]
140201394339744 -> 140201394339072
140201394339744 [label=ToCopyBackward0]
140201394339648 -> 140201394339744
140193039468720 [label="encoder.layer.7.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039468720 -> 140201394339648
140201394339648 [label=AccumulateGrad]
140201394337776 -> 140193036629328
140201394338832 -> 140193036629856
140193039468480 [label="encoder.layer.7.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039468480 -> 140201394338832
140201394338832 [label=AccumulateGrad]
140201394338496 -> 140193036629856
140193039468560 [label="encoder.layer.7.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039468560 -> 140201394338496
140201394338496 [label=AccumulateGrad]
140193036628032 -> 140193036628176
140193036628032 [label=TBackward0]
140193036628416 -> 140193036628032
140193036628416 [label=ToCopyBackward0]
140193036629520 -> 140193036628416
140193039467360 [label="encoder.layer.8.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039467360 -> 140193036629520
140193036629520 [label=AccumulateGrad]
140193036594144 -> 140193036594288
140193036594144 [label=UnsafeViewBackward0]
140193036594960 -> 140193036594144
140193036594960 [label=CloneBackward0]
140193036595152 -> 140193036594960
140193036595152 [label=ExpandBackward0]
140193036628560 -> 140193036595152
140193036628560 [label=TransposeBackward0]
140193036628848 -> 140193036628560
140193036628848 [label=PermuteBackward0]
140193036628080 -> 140193036628848
140193036628080 [label=ViewBackward0]
140201394339216 -> 140193036628080
140201394339216 [label=ViewBackward0]
140201394339456 -> 140201394339216
140201394339456 [label=AddmmBackward0]
140201394339120 -> 140201394339456
140201394339120 [label=ToCopyBackward0]
140201394377104 -> 140201394339120
140193039467520 [label="encoder.layer.8.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039467520 -> 140201394377104
140201394377104 [label=AccumulateGrad]
140201394376912 -> 140201394339456
140201394376912 [label=ViewBackward0]
140201394377440 -> 140201394376912
140201394377440 [label=ToCopyBackward0]
140193036570576 -> 140201394377440
140201394377152 -> 140201394339456
140201394377152 [label=TBackward0]
140201394376864 -> 140201394377152
140201394376864 [label=ToCopyBackward0]
140201394377584 -> 140201394376864
140193039467600 [label="encoder.layer.8.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039467600 -> 140201394377584
140201394377584 [label=AccumulateGrad]
140193036592848 -> 140193036592656
140193036592848 [label=UnsafeViewBackward0]
140193036593184 -> 140193036592848
140193036593184 [label=CloneBackward0]
140193036593616 -> 140193036593184
140193036593616 [label=ExpandBackward0]
140193036593904 -> 140193036593616
140193036593904 [label=PermuteBackward0]
140193036593040 -> 140193036593904
140193036593040 [label=ViewBackward0]
140193036594672 -> 140193036593040
140193036594672 [label=ViewBackward0]
140193036629040 -> 140193036594672
140193036629040 [label=AddmmBackward0]
140193036592944 -> 140193036629040
140193036592944 [label=ToCopyBackward0]
140201394377344 -> 140193036592944
140193039466560 [label="encoder.layer.8.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039466560 -> 140201394377344
140201394377344 [label=AccumulateGrad]
140201394338688 -> 140193036629040
140201394338688 [label=ViewBackward0]
140201394377680 -> 140201394338688
140201394377680 [label=ToCopyBackward0]
140193036570576 -> 140201394377680
140201394338928 -> 140193036629040
140201394338928 [label=TBackward0]
140201394377248 -> 140201394338928
140201394377248 [label=ToCopyBackward0]
140201394377728 -> 140201394377248
140193039466880 [label="encoder.layer.8.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039466880 -> 140201394377728
140201394377728 [label=AccumulateGrad]
140193036591312 -> 140193036591600
140193036591312 [label=TBackward0]
140193036592368 -> 140193036591312
140193036592368 [label=ToCopyBackward0]
140193036592752 -> 140193036592368
140193039466640 [label="encoder.layer.8.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039466640 -> 140193036592752
140193036592752 [label=AccumulateGrad]
140193036570576 -> 140193036570192
140193036570288 -> 140193036570000
140193039461008 [label="encoder.layer.8.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039461008 -> 140193036570288
140193036570288 [label=AccumulateGrad]
140193036569232 -> 140193036570000
140193039460768 [label="encoder.layer.8.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039460768 -> 140193036569232
140193036569232 [label=AccumulateGrad]
140193036567840 -> 140193036568800
140193036567840 [label=TBackward0]
140193036569424 -> 140193036567840
140193036569424 [label=ToCopyBackward0]
140193036570096 -> 140193036569424
140193039460448 [label="encoder.layer.8.crossattention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039460448 -> 140193036570096
140193036570096 [label=AccumulateGrad]
140193036567792 -> 140193036567888
140193036567792 [label=UnsafeViewBackward0]
140193036568464 -> 140193036567792
140193036568464 [label=CloneBackward0]
140193036568848 -> 140193036568464
140193036568848 [label=ExpandBackward0]
140193036569328 -> 140193036568848
140193036569328 [label=TransposeBackward0]
140193036569904 -> 140193036569328
140193036569904 [label=PermuteBackward0]
140193036570384 -> 140193036569904
140193036570384 [label=ViewBackward0]
140193036569520 -> 140193036570384
140193036569520 [label=ViewBackward0]
140193036591696 -> 140193036569520
140193036591696 [label=AddmmBackward0]
140193036592560 -> 140193036591696
140193036592560 [label=ToCopyBackward0]
140193036593424 -> 140193036592560
140193039459968 [label="encoder.layer.8.crossattention.self.key.bias
(768)" fillcolor=lightblue]
140193039459968 -> 140193036593424
140193036593424 [label=AccumulateGrad]
140193036591888 -> 140193036591696
140193036591888 [label=ViewBackward0]
140193036593664 -> 140193036591888
140193036593664 [label=ToCopyBackward0]
140193036594768 -> 140193036593664
140193036594768 [label=ViewBackward0]
140201394339408 -> 140193036594768
140201394339408 [label=CloneBackward0]
140193036592272 -> 140201394339408
140193036592272 [label=ExpandBackward0]
140201394377776 -> 140193036592272
140201394377776 [label=UnsqueezeBackward0]
140193036037232 -> 140201394377776
140193036591216 -> 140193036591696
140193036591216 [label=TBackward0]
140193036628272 -> 140193036591216
140193036628272 [label=ToCopyBackward0]
140193036594096 -> 140193036628272
140193039460288 [label="encoder.layer.8.crossattention.self.key.weight
(768, 1408)" fillcolor=lightblue]
140193039460288 -> 140193036594096
140193036594096 [label=AccumulateGrad]
140193036537712 -> 140193036537424
140193036537712 [label=UnsafeViewBackward0]
140193036537808 -> 140193036537712
140193036537808 [label=CloneBackward0]
140193036567120 -> 140193036537808
140193036567120 [label=ExpandBackward0]
140193036567360 -> 140193036567120
140193036567360 [label=PermuteBackward0]
140193036566736 -> 140193036567360
140193036566736 [label=ViewBackward0]
140193036568656 -> 140193036566736
140193036568656 [label=ViewBackward0]
140193036569808 -> 140193036568656
140193036569808 [label=AddmmBackward0]
140193036568176 -> 140193036569808
140193036568176 [label=ToCopyBackward0]
140193036591744 -> 140193036568176
140193039459728 [label="encoder.layer.8.crossattention.self.value.bias
(768)" fillcolor=lightblue]
140193039459728 -> 140193036591744
140193036591744 [label=AccumulateGrad]
140193036570240 -> 140193036569808
140193036570240 [label=ViewBackward0]
140193036593136 -> 140193036570240
140193036593136 [label=ToCopyBackward0]
140193036594768 -> 140193036593136
140193036566640 -> 140193036569808
140193036566640 [label=TBackward0]
140201394377824 -> 140193036566640
140201394377824 [label=ToCopyBackward0]
140201394377488 -> 140201394377824
140193039460048 [label="encoder.layer.8.crossattention.self.value.weight
(768, 1408)" fillcolor=lightblue]
140193039460048 -> 140201394377488
140201394377488 [label=AccumulateGrad]
140193036536080 -> 140193036536368
140193036536080 [label=TBackward0]
140193036537232 -> 140193036536080
140193036537232 [label=ToCopyBackward0]
140193036537520 -> 140193036537232
140193039459808 [label="encoder.layer.8.crossattention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039459808 -> 140193036537520
140193036537520 [label=AccumulateGrad]
140193036535888 -> 140193036535504
140193036535600 -> 140193036535264
140193039459568 [label="encoder.layer.8.crossattention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039459568 -> 140193036535600
140193036535600 [label=AccumulateGrad]
140193036535024 -> 140193036535264
140193039459248 [label="encoder.layer.8.crossattention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039459248 -> 140193036535024
140193036535024 [label=AccumulateGrad]
140193036533872 -> 140193036534256
140193036533872 [label=TBackward0]
140193036535312 -> 140193036533872
140193036535312 [label=ToCopyBackward0]
140193036535408 -> 140193036535312
140193039443744 [label="encoder.layer.8.experts.experts.0.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039443744 -> 140193036535408
140193036535408 [label=AccumulateGrad]
140193036516560 -> 140193036517040
140193036516560 [label=TBackward0]
140193036534352 -> 140193036516560
140193036534352 [label=ToCopyBackward0]
140193036534544 -> 140193036534352
140193039443824 [label="encoder.layer.8.experts.experts.0.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039443824 -> 140193036534544
140193036534544 [label=AccumulateGrad]
140193036516464 -> 140193036516176
140193036516464 [label=UnsqueezeBackward0]
140193036516848 -> 140193036516464
140193036516848 [label=NativeDropoutBackward0]
140193036517136 -> 140193036516848
140193036517136 [label=ViewBackward0]
140193036535696 -> 140193036517136
140193036535696 [label=AddmmBackward0]
140193036534160 -> 140193036535696
140193036534160 [label=ToCopyBackward0]
140193036536176 -> 140193036534160
140193039443504 [label="encoder.layer.8.experts.experts.1.dense2.bias
(768)" fillcolor=lightblue]
140193039443504 -> 140193036536176
140193036536176 [label=AccumulateGrad]
140193036534832 -> 140193036535696
140193036534832 [label=ViewBackward0]
140193036536464 -> 140193036534832
140193036536464 [label=GeluBackward0]
140193036537328 -> 140193036536464
140193036537328 [label=ViewBackward0]
140193036537040 -> 140193036537328
140193036537040 [label=AddmmBackward0]
140193036535984 -> 140193036537040
140193036535984 [label=ToCopyBackward0]
140193036568944 -> 140193036535984
140193039444304 [label="encoder.layer.8.experts.experts.1.dense1.bias
(3072)" fillcolor=lightblue]
140193039444304 -> 140193036568944
140193036568944 [label=AccumulateGrad]
140193036566880 -> 140193036537040
140193036566880 [label=ViewBackward0]
140193036567600 -> 140193036566880
140193036567600 [label=ToCopyBackward0]
140193036535120 -> 140193036567600
140193036566832 -> 140193036537040
140193036566832 [label=TBackward0]
140193036591264 -> 140193036566832
140193036591264 [label=ToCopyBackward0]
140201394377968 -> 140193036591264
140193039443584 [label="encoder.layer.8.experts.experts.1.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039443584 -> 140201394377968
140201394377968 [label=AccumulateGrad]
140193036533824 -> 140193036535696
140193036533824 [label=TBackward0]
140193036536656 -> 140193036533824
140193036536656 [label=ToCopyBackward0]
140193036568368 -> 140193036536656
140193039443344 [label="encoder.layer.8.experts.experts.1.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039443344 -> 140193036568368
140193036568368 [label=AccumulateGrad]
140193036515696 -> 140193036515792
140193036515696 [label=UnsqueezeBackward0]
140193036515888 -> 140193036515696
140193036515888 [label=UnsqueezeBackward0]
140193036516416 -> 140193036515888
140193036516416 [label=MulBackward0]
140193036567312 -> 140193036516416
140193036567312 [label=IndexBackward0]
140193036534784 -> 140193036567312
140193036534784 [label=SoftmaxBackward0]
140193036536752 -> 140193036534784
140193036536752 [label=CatBackward0]
140201394377920 -> 140193036536752
140201394377920 [label=MmBackward0]
140201394378064 -> 140201394377920
140201394378064 [label=MeanBackward1]
140193036516752 -> 140201394378064
140201394378016 -> 140201394377920
140201394378016 [label=TBackward0]
140201394378112 -> 140201394378016
140201394378112 [label=ToCopyBackward0]
140201394378304 -> 140201394378112
140193039444704 [label="encoder.layer.8.experts.gate.weight
(1, 768)" fillcolor=lightblue]
140193039444704 -> 140201394378304
140201394378304 [label=AccumulateGrad]
140201394377536 -> 140193036536752
140201394377536 [label=MmBackward0]
140201394378256 -> 140201394377536
140201394378256 [label=MeanBackward1]
140193036516848 -> 140201394378256
140201394378160 -> 140201394377536
140201394378160 [label=TBackward0]
140201394378112 -> 140201394378160
140193036515024 -> 140193036514640
140193036515024 [label=UnsqueezeBackward0]
140193036515408 -> 140193036515024
140193036515408 [label=SelectBackward0]
140193036515120 -> 140193036515408
140193036515120 [label=NativeDropoutBackward0]
140193036516656 -> 140193036515120
140193036516656 [label=ViewBackward0]
140193036516080 -> 140193036516656
140193036516080 [label=AddmmBackward0]
140193036534640 -> 140193036516080
140193036534640 [label=ToCopyBackward0]
140201394378208 -> 140193036534640
140193039443024 [label="encoder.layer.8.experts.experts.2.dense2.bias
(768)" fillcolor=lightblue]
140193039443024 -> 140201394378208
140201394378208 [label=AccumulateGrad]
140193036535792 -> 140193036516080
140193036535792 [label=ViewBackward0]
140201394378448 -> 140193036535792
140201394378448 [label=GeluBackward0]
140201394378544 -> 140201394378448
140201394378544 [label=ViewBackward0]
140201394378640 -> 140201394378544
140201394378640 [label=AddmmBackward0]
140201394378736 -> 140201394378640
140201394378736 [label=ToCopyBackward0]
140201394378928 -> 140201394378736
140193039443264 [label="encoder.layer.8.experts.experts.2.dense1.bias
(3072)" fillcolor=lightblue]
140193039443264 -> 140201394378928
140201394378928 [label=AccumulateGrad]
140201394378688 -> 140201394378640
140201394378688 [label=ViewBackward0]
140201394378976 -> 140201394378688
140201394378976 [label=ToCopyBackward0]
140201394379072 -> 140201394378976
140201394379072 [label=IndexBackward0]
140193036514352 -> 140201394379072
140201394378352 -> 140201394378640
140201394378352 [label=TBackward0]
140201394379168 -> 140201394378352
140201394379168 [label=ToCopyBackward0]
140201394378880 -> 140201394379168
140193039443104 [label="encoder.layer.8.experts.experts.2.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039443104 -> 140201394378880
140201394378880 [label=AccumulateGrad]
140201394377632 -> 140193036516080
140201394377632 [label=TBackward0]
140201394378592 -> 140201394377632
140201394378592 [label=ToCopyBackward0]
140201394378832 -> 140201394378592
140193039442864 [label="encoder.layer.8.experts.experts.2.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039442864 -> 140201394378832
140201394378832 [label=AccumulateGrad]
140193036514448 -> 140193036514256
140193036514448 [label=CatBackward0]
140193036515456 -> 140193036514448
140193036515456 [label=SliceBackward0]
140193036514976 -> 140193036515456
140193036514976 [label=SliceBackward0]
140201394378784 -> 140193036514976
140201394378784 [label=SliceBackward0]
140193036515600 -> 140201394378784
140193036515216 -> 140193036514448
140193036515216 [label=UnsqueezeBackward0]
140193036516272 -> 140193036515216
140193036516272 [label=SelectBackward0]
140193036515120 -> 140193036516272
140193036514544 -> 140193036514256
140193036514544 [label=CatBackward0]
140193036514736 -> 140193036514544
140193036514736 [label=SliceBackward0]
140201394378496 -> 140193036514736
140201394378496 [label=SliceBackward0]
140201394379024 -> 140201394378496
140201394379024 [label=SliceBackward0]
140193036515600 -> 140201394379024
140201394377872 -> 140193036514544
140201394377872 [label=UnsqueezeBackward0]
140201394379360 -> 140201394377872
140201394379360 [label=SelectBackward0]
140193036515120 -> 140201394379360
140193036514496 -> 140193036514256
140193036514496 [label=CatBackward0]
140201394379264 -> 140193036514496
140201394379264 [label=SliceBackward0]
140201394379408 -> 140201394379264
140201394379408 [label=SliceBackward0]
140201394379504 -> 140201394379408
140201394379504 [label=SliceBackward0]
140193036515600 -> 140201394379504
140201394379216 -> 140193036514496
140201394379216 [label=UnsqueezeBackward0]
140201394379600 -> 140201394379216
140201394379600 [label=SelectBackward0]
140193036515120 -> 140201394379600
140193036514352 -> 140193036513968
140193036514064 -> 140193036513536
140193039445024 [label="encoder.layer.8.expert_ln.weight
(768)" fillcolor=lightblue]
140193039445024 -> 140193036514064
140193036514064 [label=AccumulateGrad]
140193036513776 -> 140193036513536
140193039444784 [label="encoder.layer.8.expert_ln.bias
(768)" fillcolor=lightblue]
140193039444784 -> 140193036513776
140193036513776 [label=AccumulateGrad]
140193036513392 -> 140193036975408
140193036513392 [label=IndexBackward0]
140193036514832 -> 140193036513392
140193036514832 [label=NativeLayerNormBackward0]
140193036514160 -> 140193036514832
140193036514160 [label=AddBackward0]
140201394379648 -> 140193036514160
140201394379648 [label=NativeDropoutBackward0]
140201394379792 -> 140201394379648
140201394379792 [label=ViewBackward0]
140201394379888 -> 140201394379792
140201394379888 [label=AddmmBackward0]
140201394379984 -> 140201394379888
140201394379984 [label=ToCopyBackward0]
140201394380176 -> 140201394379984
140193039458768 [label="encoder.layer.8.output.dense.bias
(768)" fillcolor=lightblue]
140193039458768 -> 140201394380176
140201394380176 [label=AccumulateGrad]
140201394379936 -> 140201394379888
140201394379936 [label=ViewBackward0]
140201394380224 -> 140201394379936
140201394380224 [label=GeluBackward0]
140201394380320 -> 140201394380224
140201394380320 [label=ViewBackward0]
140201394380416 -> 140201394380320
140201394380416 [label=AddmmBackward0]
140201394380512 -> 140201394380416
140201394380512 [label=ToCopyBackward0]
140201394380704 -> 140201394380512
140193039459008 [label="encoder.layer.8.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039459008 -> 140201394380704
140201394380704 [label=AccumulateGrad]
140201394380464 -> 140201394380416
140201394380464 [label=ViewBackward0]
140201394380608 -> 140201394380464
140201394380608 [label=ToCopyBackward0]
140201394378400 -> 140201394380608
140201394378400 [label=SliceBackward0]
140201394430112 -> 140201394378400
140201394430112 [label=SliceBackward0]
140201394430208 -> 140201394430112
140201394430208 [label=SliceBackward0]
140193036570000 -> 140201394430208
140201394380128 -> 140201394380416
140201394380128 [label=TBackward0]
140201394380656 -> 140201394380128
140201394380656 [label=ToCopyBackward0]
140201394430304 -> 140201394380656
140193039459328 [label="encoder.layer.8.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039459328 -> 140201394430304
140201394430304 [label=AccumulateGrad]
140201394379696 -> 140201394379888
140201394379696 [label=TBackward0]
140201394380368 -> 140201394379696
140201394380368 [label=ToCopyBackward0]
140201394380752 -> 140201394380368
140193039459088 [label="encoder.layer.8.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039459088 -> 140201394380752
140201394380752 [label=AccumulateGrad]
140201394378400 -> 140193036514160
140201394379456 -> 140193036514832
140193039458848 [label="encoder.layer.8.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039458848 -> 140201394379456
140201394379456 [label=AccumulateGrad]
140201394379120 -> 140193036514832
140193039458528 [label="encoder.layer.8.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039458528 -> 140201394379120
140201394379120 [label=AccumulateGrad]
140193036483120 -> 140193036484080
140193036483120 [label=TBackward0]
140193036513344 -> 140193036483120
140193036513344 [label=ToCopyBackward0]
140193036514016 -> 140193036513344
140193039444944 [label="encoder.layer.9.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039444944 -> 140193036514016
140193036514016 [label=AccumulateGrad]
140193036482880 -> 140193036483024
140193036482880 [label=UnsafeViewBackward0]
140193036483696 -> 140193036482880
140193036483696 [label=CloneBackward0]
140193036483984 -> 140193036483696
140193036483984 [label=ExpandBackward0]
140193036484464 -> 140193036483984
140193036484464 [label=TransposeBackward0]
140193036513488 -> 140193036484464
140193036513488 [label=PermuteBackward0]
140193036483408 -> 140193036513488
140193036483408 [label=ViewBackward0]
140201394379840 -> 140193036483408
140201394379840 [label=ViewBackward0]
140201394380080 -> 140201394379840
140201394380080 [label=AddmmBackward0]
140201394380272 -> 140201394380080
140201394380272 [label=ToCopyBackward0]
140201394430064 -> 140201394380272
140193039445504 [label="encoder.layer.9.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039445504 -> 140201394430064
140201394430064 [label=AccumulateGrad]
140201394380560 -> 140201394380080
140201394380560 [label=ViewBackward0]
140201394430352 -> 140201394380560
140201394430352 [label=ToCopyBackward0]
140193036975408 -> 140201394430352
140201394379744 -> 140201394380080
140201394379744 [label=TBackward0]
140201394430160 -> 140201394379744
140201394430160 [label=ToCopyBackward0]
140201394430496 -> 140201394430160
140193039445184 [label="encoder.layer.9.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039445184 -> 140201394430496
140201394430496 [label=AccumulateGrad]
140193036481584 -> 140193036481392
140193036481584 [label=UnsafeViewBackward0]
140193036481920 -> 140193036481584
140193036481920 [label=CloneBackward0]
140193036482352 -> 140193036481920
140193036482352 [label=ExpandBackward0]
140193036482640 -> 140193036482352
140193036482640 [label=PermuteBackward0]
140193036481776 -> 140193036482640
140193036481776 [label=ViewBackward0]
140193036483888 -> 140193036481776
140193036483888 [label=ViewBackward0]
140193036513680 -> 140193036483888
140193036513680 [label=AddmmBackward0]
140193036481680 -> 140193036513680
140193036481680 [label=ToCopyBackward0]
140201394430256 -> 140193036481680
140193039444544 [label="encoder.layer.9.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039444544 -> 140201394430256
140201394430256 [label=AccumulateGrad]
140201394379312 -> 140193036513680
140201394379312 [label=ViewBackward0]
140201394430592 -> 140201394379312
140201394430592 [label=ToCopyBackward0]
140193036975408 -> 140201394430592
140201394379552 -> 140193036513680
140201394379552 [label=TBackward0]
140201394430016 -> 140201394379552
140201394430016 [label=ToCopyBackward0]
140201394430640 -> 140201394430016
140193039444464 [label="encoder.layer.9.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039444464 -> 140201394430640
140201394430640 [label=AccumulateGrad]
140193036975600 -> 140193036975888
140193036975600 [label=TBackward0]
140193036481104 -> 140193036975600
140193036481104 [label=ToCopyBackward0]
140193036481488 -> 140193036481104
140193039444224 [label="encoder.layer.9.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039444224 -> 140193036481488
140193036481488 [label=AccumulateGrad]
140193036975408 -> 140193036975024
140193036975120 -> 140193036974832
140193039442784 [label="encoder.layer.9.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039442784 -> 140193036975120
140193036975120 [label=AccumulateGrad]
140193036974064 -> 140193036974832
140193039442544 [label="encoder.layer.9.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039442544 -> 140193036974064
140193036974064 [label=AccumulateGrad]
140193036972672 -> 140193036973152
140193036972672 [label=TBackward0]
140193036974256 -> 140193036972672
140193036974256 [label=ToCopyBackward0]
140193036974448 -> 140193036974256
140193039431216 [label="encoder.layer.9.experts.experts.0.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039431216 -> 140193036974448
140193036974448 [label=AccumulateGrad]
140193036972240 -> 140193036972528
140193036972240 [label=TBackward0]
140193036973296 -> 140193036972240
140193036973296 [label=ToCopyBackward0]
140193036973584 -> 140193036973296
140193039430896 [label="encoder.layer.9.experts.experts.0.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039430896 -> 140193036973584
140193036973584 [label=AccumulateGrad]
140193036951312 -> 140193036951120
140193036951312 [label=UnsqueezeBackward0]
140193036951504 -> 140193036951312
140193036951504 [label=NativeDropoutBackward0]
140193036972816 -> 140193036951504
140193036972816 [label=ViewBackward0]
140193036974736 -> 140193036972816
140193036974736 [label=AddmmBackward0]
140193036973200 -> 140193036974736
140193036973200 [label=ToCopyBackward0]
140193036975216 -> 140193036973200
140193039430976 [label="encoder.layer.9.experts.experts.1.dense2.bias
(768)" fillcolor=lightblue]
140193039430976 -> 140193036975216
140193036975216 [label=AccumulateGrad]
140193036973776 -> 140193036974736
140193036973776 [label=ViewBackward0]
140193036975072 -> 140193036973776
140193036975072 [label=GeluBackward0]
140193036975552 -> 140193036975072
140193036975552 [label=ViewBackward0]
140193036975984 -> 140193036975552
140193036975984 [label=AddmmBackward0]
140193036481872 -> 140193036975984
140193036481872 [label=ToCopyBackward0]
140193036482400 -> 140193036481872
140193039431376 [label="encoder.layer.9.experts.experts.1.dense1.bias
(3072)" fillcolor=lightblue]
140193039431376 -> 140193036482400
140193036482400 [label=AccumulateGrad]
140193036481296 -> 140193036975984
140193036481296 [label=ViewBackward0]
140193036482832 -> 140193036481296
140193036482832 [label=ToCopyBackward0]
140193036974160 -> 140193036482832
140193036480576 -> 140193036975984
140193036480576 [label=TBackward0]
140193036481008 -> 140193036480576
140193036481008 [label=ToCopyBackward0]
140201394380032 -> 140193036481008
140193039430656 [label="encoder.layer.9.experts.experts.1.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039430656 -> 140201394380032
140201394380032 [label=AccumulateGrad]
140193036972144 -> 140193036974736
140193036972144 [label=TBackward0]
140193036974928 -> 140193036972144
140193036974928 [label=ToCopyBackward0]
140193036975504 -> 140193036974928
140193039430416 [label="encoder.layer.9.experts.experts.1.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039430416 -> 140193036975504
140193036975504 [label=AccumulateGrad]
140193036950640 -> 140193036950736
140193036950640 [label=UnsqueezeBackward0]
140193036950688 -> 140193036950640
140193036950688 [label=UnsqueezeBackward0]
140193036951024 -> 140193036950688
140193036951024 [label=MulBackward0]
140193036974640 -> 140193036951024
140193036974640 [label=IndexBackward0]
140193036973872 -> 140193036974640
140193036973872 [label=SoftmaxBackward0]
140193036482160 -> 140193036973872
140193036482160 [label=CatBackward0]
140193036480624 -> 140193036482160
140193036480624 [label=MmBackward0]
140201394430688 -> 140193036480624
140201394430688 [label=MeanBackward1]
140193036951408 -> 140201394430688
140201394430736 -> 140193036480624
140201394430736 [label=TBackward0]
140201394430544 -> 140201394430736
140201394430544 [label=ToCopyBackward0]
140201394430928 -> 140201394430544
140193039432176 [label="encoder.layer.9.experts.gate.weight
(1, 768)" fillcolor=lightblue]
140193039432176 -> 140201394430928
140201394430928 [label=AccumulateGrad]
140193036484176 -> 140193036482160
140193036484176 [label=MmBackward0]
140201394430880 -> 140193036484176
140201394430880 [label=MeanBackward1]
140193036951504 -> 140201394430880
140201394430784 -> 140193036484176
140201394430784 [label=TBackward0]
140201394430544 -> 140201394430784
140193036949872 -> 140193036949584
140193036949872 [label=UnsqueezeBackward0]
140193036950208 -> 140193036949872
140193036950208 [label=SelectBackward0]
140193036950064 -> 140193036950208
140193036950064 [label=NativeDropoutBackward0]
140193036951216 -> 140193036950064
140193036951216 [label=ViewBackward0]
140193036972192 -> 140193036951216
140193036972192 [label=AddmmBackward0]
140193036483504 -> 140193036972192
140193036483504 [label=ToCopyBackward0]
140201394430832 -> 140193036483504
140193039430496 [label="encoder.layer.9.experts.experts.2.dense2.bias
(768)" fillcolor=lightblue]
140193039430496 -> 140201394430832
140201394430832 [label=AccumulateGrad]
140193036972624 -> 140193036972192
140193036972624 [label=ViewBackward0]
140201394431072 -> 140193036972624
140201394431072 [label=GeluBackward0]
140201394431168 -> 140201394431072
140201394431168 [label=ViewBackward0]
140201394431264 -> 140201394431168
140201394431264 [label=AddmmBackward0]
140201394431360 -> 140201394431264
140201394431360 [label=ToCopyBackward0]
140201394431552 -> 140201394431360
140193039430736 [label="encoder.layer.9.experts.experts.2.dense1.bias
(3072)" fillcolor=lightblue]
140193039430736 -> 140201394431552
140201394431552 [label=AccumulateGrad]
140201394431312 -> 140201394431264
140201394431312 [label=ViewBackward0]
140201394431600 -> 140201394431312
140201394431600 [label=ToCopyBackward0]
140201394431696 -> 140201394431600
140201394431696 [label=IndexBackward0]
140193036949296 -> 140201394431696
140201394430976 -> 140201394431264
140201394430976 [label=TBackward0]
140201394431792 -> 140201394430976
140201394431792 [label=ToCopyBackward0]
140201394431504 -> 140201394431792
140193039430176 [label="encoder.layer.9.experts.experts.2.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039430176 -> 140201394431504
140201394431504 [label=AccumulateGrad]
140201394430448 -> 140193036972192
140201394430448 [label=TBackward0]
140201394431216 -> 140201394430448
140201394431216 [label=ToCopyBackward0]
140201394431456 -> 140201394431216
140193039429936 [label="encoder.layer.9.experts.experts.2.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039429936 -> 140201394431456
140201394431456 [label=AccumulateGrad]
140193036949248 -> 140193036949200
140193036949248 [label=CatBackward0]
140193036950448 -> 140193036949248
140193036950448 [label=SliceBackward0]
140193036973680 -> 140193036950448
140193036973680 [label=SliceBackward0]
140201394431408 -> 140193036973680
140201394431408 [label=SliceBackward0]
140193036950544 -> 140201394431408
140193036950160 -> 140193036949248
140193036950160 [label=UnsqueezeBackward0]
140193036949968 -> 140193036950160
140193036949968 [label=SelectBackward0]
140193036950064 -> 140193036949968
140193036949392 -> 140193036949200
140193036949392 [label=CatBackward0]
140193036949680 -> 140193036949392
140193036949680 [label=SliceBackward0]
140201394431120 -> 140193036949680
140201394431120 [label=SliceBackward0]
140201394431648 -> 140201394431120
140201394431648 [label=SliceBackward0]
140193036950544 -> 140201394431648
140201394430400 -> 140193036949392
140201394430400 [label=UnsqueezeBackward0]
140201394431984 -> 140201394430400
140201394431984 [label=SelectBackward0]
140193036950064 -> 140201394431984
140193036949488 -> 140193036949200
140193036949488 [label=CatBackward0]
140201394431888 -> 140193036949488
140201394431888 [label=SliceBackward0]
140201394432032 -> 140201394431888
140201394432032 [label=SliceBackward0]
140201394432128 -> 140201394432032
140201394432128 [label=SliceBackward0]
140193036950544 -> 140201394432128
140201394431840 -> 140193036949488
140201394431840 [label=UnsqueezeBackward0]
140201394432224 -> 140201394431840
140201394432224 [label=SelectBackward0]
140193036950064 -> 140201394432224
140193036949296 -> 140193036948768
140193036948912 -> 140193036948816
140193039432096 [label="encoder.layer.9.expert_ln.weight
(768)" fillcolor=lightblue]
140193039432096 -> 140193036948912
140193036948912 [label=AccumulateGrad]
140193036948720 -> 140193036948816
140193039431856 [label="encoder.layer.9.expert_ln.bias
(768)" fillcolor=lightblue]
140193039431856 -> 140193036948720
140193036948720 [label=AccumulateGrad]
140193036948240 -> 140193036914800
140193036948240 [label=IndexBackward0]
140193036949776 -> 140193036948240
140193036949776 [label=NativeLayerNormBackward0]
140193036949104 -> 140193036949776
140193036949104 [label=AddBackward0]
140201394432272 -> 140193036949104
140201394432272 [label=NativeDropoutBackward0]
140201394432416 -> 140201394432272
140201394432416 [label=ViewBackward0]
140201394432512 -> 140201394432416
140201394432512 [label=AddmmBackward0]
140201394432608 -> 140201394432512
140201394432608 [label=ToCopyBackward0]
140201394432800 -> 140201394432608
140193039442144 [label="encoder.layer.9.output.dense.bias
(768)" fillcolor=lightblue]
140193039442144 -> 140201394432800
140201394432800 [label=AccumulateGrad]
140201394432560 -> 140201394432512
140201394432560 [label=ViewBackward0]
140201394432848 -> 140201394432560
140201394432848 [label=GeluBackward0]
140201394432944 -> 140201394432848
140201394432944 [label=ViewBackward0]
140201394433040 -> 140201394432944
140201394433040 [label=AddmmBackward0]
140201394433136 -> 140201394433040
140201394433136 [label=ToCopyBackward0]
140201394433328 -> 140201394433136
140193039442384 [label="encoder.layer.9.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039442384 -> 140201394433328
140201394433328 [label=AccumulateGrad]
140201394433088 -> 140201394433040
140201394433088 [label=ViewBackward0]
140201394433376 -> 140201394433088
140201394433376 [label=ToCopyBackward0]
140201394431024 -> 140201394433376
140201394431024 [label=SliceBackward0]
140201394433520 -> 140201394431024
140201394433520 [label=SliceBackward0]
140201394433616 -> 140201394433520
140201394433616 [label=SliceBackward0]
140193036974832 -> 140201394433616
140201394432752 -> 140201394433040
140201394432752 [label=TBackward0]
140201394433280 -> 140201394432752
140201394433280 [label=ToCopyBackward0]
140201394433712 -> 140201394433280
140193039442624 [label="encoder.layer.9.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039442624 -> 140201394433712
140201394433712 [label=AccumulateGrad]
140201394432320 -> 140201394432512
140201394432320 [label=TBackward0]
140201394432992 -> 140201394432320
140201394432992 [label=ToCopyBackward0]
140201394433472 -> 140201394432992
140193039442064 [label="encoder.layer.9.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039442064 -> 140201394433472
140201394433472 [label=AccumulateGrad]
140201394431024 -> 140193036949104
140201394432080 -> 140193036949776
140193039433536 [label="encoder.layer.9.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039433536 -> 140201394432080
140201394432080 [label=AccumulateGrad]
140201394431744 -> 140193036949776
140193039433616 [label="encoder.layer.9.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039433616 -> 140201394431744
140201394431744 [label=AccumulateGrad]
140193036947664 -> 140193036947952
140193036947664 [label=TBackward0]
140193036948336 -> 140193036947664
140193036948336 [label=ToCopyBackward0]
140193036949008 -> 140193036948336
140193039432416 [label="encoder.layer.10.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039432416 -> 140193036949008
140193036949008 [label=AccumulateGrad]
140193036918160 -> 140193036917872
140193036918160 [label=UnsafeViewBackward0]
140193036918544 -> 140193036918160
140193036918544 [label=CloneBackward0]
140193036918256 -> 140193036918544
140193036918256 [label=ExpandBackward0]
140193036948048 -> 140193036918256
140193036948048 [label=TransposeBackward0]
140193036948624 -> 140193036948048
140193036948624 [label=PermuteBackward0]
140193036947568 -> 140193036948624
140193036947568 [label=ViewBackward0]
140201394432464 -> 140193036947568
140201394432464 [label=ViewBackward0]
140201394432704 -> 140201394432464
140201394432704 [label=AddmmBackward0]
140201394433232 -> 140201394432704
140201394433232 [label=ToCopyBackward0]
140201394433424 -> 140201394433232
140193039432576 [label="encoder.layer.10.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039432576 -> 140201394433424
140201394433424 [label=AccumulateGrad]
140201394433184 -> 140201394432704
140201394433184 [label=ViewBackward0]
140201394433760 -> 140201394433184
140201394433760 [label=ToCopyBackward0]
140193036914800 -> 140201394433760
140201394432368 -> 140201394432704
140201394432368 [label=TBackward0]
140201394432896 -> 140201394432368
140201394432896 [label=ToCopyBackward0]
140201394433904 -> 140201394432896
140193039432656 [label="encoder.layer.10.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039432656 -> 140201394433904
140201394433904 [label=AccumulateGrad]
140193036916432 -> 140193036916528
140193036916432 [label=UnsafeViewBackward0]
140193036917200 -> 140193036916432
140193036917200 [label=CloneBackward0]
140193036917488 -> 140193036917200
140193036917488 [label=ExpandBackward0]
140193036917776 -> 140193036917488
140193036917776 [label=PermuteBackward0]
140193036916624 -> 140193036917776
140193036916624 [label=ViewBackward0]
140193036918112 -> 140193036916624
140193036918112 [label=ViewBackward0]
140193036948528 -> 140193036918112
140193036948528 [label=AddmmBackward0]
140193036916816 -> 140193036948528
140193036916816 [label=ToCopyBackward0]
140201394433664 -> 140193036916816
140193039431616 [label="encoder.layer.10.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039431616 -> 140201394433664
140201394433664 [label=AccumulateGrad]
140201394431936 -> 140193036948528
140201394431936 [label=ViewBackward0]
140201394434000 -> 140201394431936
140201394434000 [label=ToCopyBackward0]
140193036914800 -> 140201394434000
140201394432176 -> 140193036948528
140201394432176 [label=TBackward0]
140201394433568 -> 140201394432176
140201394433568 [label=ToCopyBackward0]
140201394433856 -> 140201394433568
140193039431936 [label="encoder.layer.10.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039431936 -> 140201394433856
140201394433856 [label=AccumulateGrad]
140193036914896 -> 140193036915184
140193036914896 [label=TBackward0]
140193036915952 -> 140193036914896
140193036915952 [label=ToCopyBackward0]
140193036916192 -> 140193036915952
140193039431696 [label="encoder.layer.10.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039431696 -> 140193036916192
140193036916192 [label=AccumulateGrad]
140193036914800 -> 140193036894064
140193036893728 -> 140193036893872
140193039430256 [label="encoder.layer.10.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039430256 -> 140193036893728
140193036893728 [label=AccumulateGrad]
140193036893104 -> 140193036893872
140193039430016 [label="encoder.layer.10.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039430016 -> 140193036893104
140193036893104 [label=AccumulateGrad]
140193036891856 -> 140193036892816
140193036891856 [label=TBackward0]
140193036893008 -> 140193036891856
140193036893008 [label=ToCopyBackward0]
140193036893680 -> 140193036893008
140193039429696 [label="encoder.layer.10.crossattention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039429696 -> 140193036893680
140193036893680 [label=AccumulateGrad]
140193036891664 -> 140193036891328
140193036891664 [label=UnsafeViewBackward0]
140193036892048 -> 140193036891664
140193036892048 [label=CloneBackward0]
140193036892288 -> 140193036892048
140193036892288 [label=ExpandBackward0]
140193036892768 -> 140193036892288
140193036892768 [label=TransposeBackward0]
140193036893488 -> 140193036892768
140193036893488 [label=PermuteBackward0]
140193036893392 -> 140193036893488
140193036893392 [label=ViewBackward0]
140193036891760 -> 140193036893392
140193036891760 [label=ViewBackward0]
140193036915568 -> 140193036891760
140193036915568 [label=AddmmBackward0]
140193036916144 -> 140193036915568
140193036916144 [label=ToCopyBackward0]
140193036917296 -> 140193036916144
140193039420928 [label="encoder.layer.10.crossattention.self.key.bias
(768)" fillcolor=lightblue]
140193039420928 -> 140193036917296
140193036917296 [label=AccumulateGrad]
140193036915472 -> 140193036915568
140193036915472 [label=ViewBackward0]
140193036917680 -> 140193036915472
140193036917680 [label=ToCopyBackward0]
140193036918352 -> 140193036917680
140193036918352 [label=ViewBackward0]
140193036947760 -> 140193036918352
140193036947760 [label=CloneBackward0]
140201394433952 -> 140193036947760
140201394433952 [label=ExpandBackward0]
140201394432656 -> 140201394433952
140201394432656 [label=UnsqueezeBackward0]
140193036037232 -> 140201394432656
140193036914752 -> 140193036915568
140193036914752 [label=TBackward0]
140193036915712 -> 140193036914752
140193036915712 [label=ToCopyBackward0]
140201394433808 -> 140193036915712
140193039421248 [label="encoder.layer.10.crossattention.self.key.weight
(768, 1408)" fillcolor=lightblue]
140193039421248 -> 140201394433808
140201394433808 [label=AccumulateGrad]
140193036852288 -> 140193036853104
140193036852288 [label=UnsafeViewBackward0]
140193036890704 -> 140193036852288
140193036890704 [label=CloneBackward0]
140193036890992 -> 140193036890704
140193036890992 [label=ExpandBackward0]
140193036891376 -> 140193036890992
140193036891376 [label=PermuteBackward0]
140193036890224 -> 140193036891376
140193036890224 [label=ViewBackward0]
140193036892240 -> 140193036890224
140193036892240 [label=ViewBackward0]
140193036893248 -> 140193036892240
140193036893248 [label=AddmmBackward0]
140193036893968 -> 140193036893248
140193036893968 [label=ToCopyBackward0]
140193036917968 -> 140193036893968
140193039420688 [label="encoder.layer.10.crossattention.self.value.bias
(768)" fillcolor=lightblue]
140193039420688 -> 140193036917968
140193036917968 [label=AccumulateGrad]
140193036890416 -> 140193036893248
140193036890416 [label=ViewBackward0]
140193036917008 -> 140193036890416
140193036917008 [label=ToCopyBackward0]
140193036918352 -> 140193036917008
140193036915088 -> 140193036893248
140193036915088 [label=TBackward0]
140193036915760 -> 140193036915088
140193036915760 [label=ToCopyBackward0]
140201394495696 -> 140193036915760
140193039421008 [label="encoder.layer.10.crossattention.self.value.weight
(768, 1408)" fillcolor=lightblue]
140193039421008 -> 140201394495696
140201394495696 [label=AccumulateGrad]
140193036851856 -> 140193036852048
140193036851856 [label=TBackward0]
140193036852480 -> 140193036851856
140193036852480 [label=ToCopyBackward0]
140193036852912 -> 140193036852480
140193039420768 [label="encoder.layer.10.crossattention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039420768 -> 140193036852912
140193036852912 [label=AccumulateGrad]
140193036851760 -> 140193036851616
140193036851568 -> 140193036851520
140193039420528 [label="encoder.layer.10.crossattention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039420528 -> 140193036851568
140193036851568 [label=AccumulateGrad]
140193036851328 -> 140193036851520
140193039420208 [label="encoder.layer.10.crossattention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039420208 -> 140193036851328
140193036851328 [label=AccumulateGrad]
140193036850560 -> 140193036850848
140193036850560 [label=TBackward0]
140193036851376 -> 140193036850560
140193036851376 [label=ToCopyBackward0]
140193036851472 -> 140193036851376
140193039417408 [label="encoder.layer.10.experts.experts.0.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039417408 -> 140193036851472
140193036851472 [label=AccumulateGrad]
140193036850032 -> 140193036850320
140193036850032 [label=TBackward0]
140193036850800 -> 140193036850032
140193036850800 [label=ToCopyBackward0]
140193036851040 -> 140193036850800
140193039404784 [label="encoder.layer.10.experts.experts.0.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039404784 -> 140193036851040
140193036851040 [label=AccumulateGrad]
140193036849936 -> 140193036849888
140193036849936 [label=UnsqueezeBackward0]
140193036850272 -> 140193036849936
140193036850272 [label=NativeDropoutBackward0]
140193036850512 -> 140193036850272
140193036850512 [label=ViewBackward0]
140193036851712 -> 140193036850512
140193036851712 [label=AddmmBackward0]
140193036850704 -> 140193036851712
140193036850704 [label=ToCopyBackward0]
140193036852000 -> 140193036850704
140193039403984 [label="encoder.layer.10.experts.experts.1.dense2.bias
(768)" fillcolor=lightblue]
140193039403984 -> 140193036852000
140193036852000 [label=AccumulateGrad]
140193036851088 -> 140193036851712
140193036851088 [label=ViewBackward0]
140193036852192 -> 140193036851088
140193036852192 [label=GeluBackward0]
140193036852720 -> 140193036852192
140193036852720 [label=ViewBackward0]
140193036852432 -> 140193036852720
140193036852432 [label=AddmmBackward0]
140193036890896 -> 140193036852432
140193036890896 [label=ToCopyBackward0]
140193036892528 -> 140193036890896
140193039404224 [label="encoder.layer.10.experts.experts.1.dense1.bias
(3072)" fillcolor=lightblue]
140193039404224 -> 140193036892528
140193036892528 [label=AccumulateGrad]
140193036890176 -> 140193036852432
140193036890176 [label=ViewBackward0]
140193036891472 -> 140193036890176
140193036891472 [label=ToCopyBackward0]
140193036851280 -> 140193036891472
140193036890512 -> 140193036852432
140193036890512 [label=TBackward0]
140193036915280 -> 140193036890512
140193036915280 [label=ToCopyBackward0]
140201394495792 -> 140193036915280
140193039404544 [label="encoder.layer.10.experts.experts.1.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039404544 -> 140201394495792
140201394495792 [label=AccumulateGrad]
140193036850080 -> 140193036851712
140193036850080 [label=TBackward0]
140193036851904 -> 140193036850080
140193036851904 [label=ToCopyBackward0]
140193036891808 -> 140193036851904
140193039404304 [label="encoder.layer.10.experts.experts.1.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039404304 -> 140193036891808
140193036891808 [label=AccumulateGrad]
140193036849600 -> 140193036849552
140193036849600 [label=UnsqueezeBackward0]
140193036891184 -> 140193036849600
140193036891184 [label=UnsqueezeBackward0]
140193036850176 -> 140193036891184
140193036850176 [label=MulBackward0]
140193036850992 -> 140193036850176
140193036850992 [label=IndexBackward0]
140193036851232 -> 140193036850992
140193036851232 [label=SoftmaxBackward0]
140193036852240 -> 140193036851232
140193036852240 [label=CatBackward0]
140201394495744 -> 140193036852240
140201394495744 [label=MmBackward0]
140201394495888 -> 140201394495744
140201394495888 [label=MeanBackward1]
140193036850128 -> 140201394495888
140201394495840 -> 140201394495744
140201394495840 [label=TBackward0]
140201394495936 -> 140201394495840
140201394495936 [label=ToCopyBackward0]
140201394496128 -> 140201394495936
140193039418048 [label="encoder.layer.10.experts.gate.weight
(1, 768)" fillcolor=lightblue]
140193039418048 -> 140201394496128
140201394496128 [label=AccumulateGrad]
140201394495552 -> 140193036852240
140201394495552 [label=MmBackward0]
140201394496080 -> 140201394495552
140201394496080 [label=MeanBackward1]
140193036850272 -> 140201394496080
140201394495984 -> 140201394495552
140201394495984 [label=TBackward0]
140201394495936 -> 140201394495984
140193036803808 -> 140193036803616
140193036803808 [label=UnsqueezeBackward0]
140193036849408 -> 140193036803808
140193036849408 [label=SelectBackward0]
140193036849216 -> 140193036849408
140193036849216 [label=NativeDropoutBackward0]
140193036849696 -> 140193036849216
140193036849696 [label=ViewBackward0]
140193036851664 -> 140193036849696
140193036851664 [label=AddmmBackward0]
140193036849744 -> 140193036851664
140193036849744 [label=ToCopyBackward0]
140201394496032 -> 140193036849744
140193039403504 [label="encoder.layer.10.experts.experts.2.dense2.bias
(768)" fillcolor=lightblue]
140193039403504 -> 140201394496032
140201394496032 [label=AccumulateGrad]
140193036849264 -> 140193036851664
140193036849264 [label=ViewBackward0]
140201394496272 -> 140193036849264
140201394496272 [label=GeluBackward0]
140201394496368 -> 140201394496272
140201394496368 [label=ViewBackward0]
140201394496464 -> 140201394496368
140201394496464 [label=AddmmBackward0]
140201394496560 -> 140201394496464
140201394496560 [label=ToCopyBackward0]
140201394496752 -> 140201394496560
140193039403744 [label="encoder.layer.10.experts.experts.2.dense1.bias
(3072)" fillcolor=lightblue]
140193039403744 -> 140201394496752
140201394496752 [label=AccumulateGrad]
140201394496512 -> 140201394496464
140201394496512 [label=ViewBackward0]
140201394496800 -> 140201394496512
140201394496800 [label=ToCopyBackward0]
140201394496896 -> 140201394496800
140201394496896 [label=IndexBackward0]
140193036803424 -> 140201394496896
140201394496176 -> 140201394496464
140201394496176 [label=TBackward0]
140201394496992 -> 140201394496176
140201394496992 [label=ToCopyBackward0]
140201394496704 -> 140201394496992
140193039404064 [label="encoder.layer.10.experts.experts.2.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039404064 -> 140201394496704
140201394496704 [label=AccumulateGrad]
140201394495648 -> 140193036851664
140201394495648 [label=TBackward0]
140201394496416 -> 140201394495648
140201394496416 [label=ToCopyBackward0]
140201394496656 -> 140201394496416
140193039403824 [label="encoder.layer.10.experts.experts.2.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039403824 -> 140201394496656
140201394496656 [label=AccumulateGrad]
140193036803760 -> 140193036803664
140193036803760 [label=CatBackward0]
140193036803952 -> 140193036803760
140193036803952 [label=SliceBackward0]
140193036850464 -> 140193036803952
140193036850464 [label=SliceBackward0]
140201394496608 -> 140193036850464
140201394496608 [label=SliceBackward0]
140193036849456 -> 140201394496608
140193036849504 -> 140193036803760
140193036849504 [label=UnsqueezeBackward0]
140193036849840 -> 140193036849504
140193036849840 [label=SelectBackward0]
140193036849216 -> 140193036849840
140193036803520 -> 140193036803664
140193036803520 [label=CatBackward0]
140193036849312 -> 140193036803520
140193036849312 [label=SliceBackward0]
140201394496320 -> 140193036849312
140201394496320 [label=SliceBackward0]
140201394496848 -> 140201394496320
140201394496848 [label=SliceBackward0]
140193036849456 -> 140201394496848
140201394495600 -> 140193036803520
140201394495600 [label=UnsqueezeBackward0]
140201394497184 -> 140201394495600
140201394497184 [label=SelectBackward0]
140193036849216 -> 140201394497184
140193036803856 -> 140193036803664
140193036803856 [label=CatBackward0]
140201394497088 -> 140193036803856
140201394497088 [label=SliceBackward0]
140201394497232 -> 140201394497088
140201394497232 [label=SliceBackward0]
140201394497328 -> 140201394497232
140201394497328 [label=SliceBackward0]
140193036849456 -> 140201394497328
140201394497040 -> 140193036803856
140201394497040 [label=UnsqueezeBackward0]
140201394497424 -> 140201394497040
140201394497424 [label=SelectBackward0]
140193036849216 -> 140201394497424
140193036803424 -> 140193036803472
140193036803232 -> 140193036803136
140193039418368 [label="encoder.layer.10.expert_ln.weight
(768)" fillcolor=lightblue]
140193039418368 -> 140193036803232
140193036803232 [label=AccumulateGrad]
140193036803376 -> 140193036803136
140193039418128 [label="encoder.layer.10.expert_ln.bias
(768)" fillcolor=lightblue]
140193039418128 -> 140193036803376
140193036803376 [label=AccumulateGrad]
140193036803088 -> 140193036800208
140193036803088 [label=IndexBackward0]
140193036803712 -> 140193036803088
140193036803712 [label=NativeLayerNormBackward0]
140193036803328 -> 140193036803712
140193036803328 [label=AddBackward0]
140201394497472 -> 140193036803328
140201394497472 [label=NativeDropoutBackward0]
140201394497616 -> 140201394497472
140201394497616 [label=ViewBackward0]
140201394497712 -> 140201394497616
140201394497712 [label=AddmmBackward0]
140201394497808 -> 140201394497712
140201394497808 [label=ToCopyBackward0]
140201394498000 -> 140201394497808
140193039419728 [label="encoder.layer.10.output.dense.bias
(768)" fillcolor=lightblue]
140193039419728 -> 140201394498000
140201394498000 [label=AccumulateGrad]
140201394497760 -> 140201394497712
140201394497760 [label=ViewBackward0]
140201394498048 -> 140201394497760
140201394498048 [label=GeluBackward0]
140201394498144 -> 140201394498048
140201394498144 [label=ViewBackward0]
140201394498240 -> 140201394498144
140201394498240 [label=AddmmBackward0]
140201394498336 -> 140201394498240
140201394498336 [label=ToCopyBackward0]
140201394498528 -> 140201394498336
140193039419968 [label="encoder.layer.10.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039419968 -> 140201394498528
140201394498528 [label=AccumulateGrad]
140201394498288 -> 140201394498240
140201394498288 [label=ViewBackward0]
140201394498576 -> 140201394498288
140201394498576 [label=ToCopyBackward0]
140201394496224 -> 140201394498576
140201394496224 [label=SliceBackward0]
140201394498720 -> 140201394496224
140201394498720 [label=SliceBackward0]
140201394498816 -> 140201394498720
140201394498816 [label=SliceBackward0]
140193036893872 -> 140201394498816
140201394497952 -> 140201394498240
140201394497952 [label=TBackward0]
140201394498480 -> 140201394497952
140201394498480 [label=ToCopyBackward0]
140201394498912 -> 140201394498480
140193039420288 [label="encoder.layer.10.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039420288 -> 140201394498912
140201394498912 [label=AccumulateGrad]
140201394497520 -> 140201394497712
140201394497520 [label=TBackward0]
140201394498192 -> 140201394497520
140201394498192 [label=ToCopyBackward0]
140201394498672 -> 140201394498192
140193039420048 [label="encoder.layer.10.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039420048 -> 140201394498672
140201394498672 [label=AccumulateGrad]
140201394496224 -> 140193036803328
140201394497280 -> 140193036803712
140193039419808 [label="encoder.layer.10.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039419808 -> 140201394497280
140201394497280 [label=AccumulateGrad]
140201394496944 -> 140193036803712
140193039419488 [label="encoder.layer.10.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039419488 -> 140201394496944
140201394496944 [label=AccumulateGrad]
140193036802080 -> 140193036802656
140193036802080 [label=TBackward0]
140193036802848 -> 140193036802080
140193036802848 [label=ToCopyBackward0]
140193036803568 -> 140193036802848
140193039418288 [label="encoder.layer.11.attention.self.query.weight
(768, 768)" fillcolor=lightblue]
140193039418288 -> 140193036803568
140193036803568 [label=AccumulateGrad]
140193036801984 -> 140193036802128
140193036801984 [label=UnsafeViewBackward0]
140193036802512 -> 140193036801984
140193036802512 [label=CloneBackward0]
140193036802704 -> 140193036802512
140193036802704 [label=ExpandBackward0]
140193036802992 -> 140193036802704
140193036802992 [label=TransposeBackward0]
140193036803040 -> 140193036802992
140193036803040 [label=PermuteBackward0]
140193036802320 -> 140193036803040
140193036802320 [label=ViewBackward0]
140201394497664 -> 140193036802320
140201394497664 [label=ViewBackward0]
140201394497904 -> 140201394497664
140201394497904 [label=AddmmBackward0]
140201394498432 -> 140201394497904
140201394498432 [label=ToCopyBackward0]
140201394498624 -> 140201394498432
140193039418848 [label="encoder.layer.11.attention.self.key.bias
(768)" fillcolor=lightblue]
140193039418848 -> 140201394498624
140201394498624 [label=AccumulateGrad]
140201394498384 -> 140201394497904
140201394498384 [label=ViewBackward0]
140201394498960 -> 140201394498384
140201394498960 [label=ToCopyBackward0]
140193036800208 -> 140201394498960
140201394497568 -> 140201394497904
140201394497568 [label=TBackward0]
140201394498096 -> 140201394497568
140201394498096 [label=ToCopyBackward0]
140201394499104 -> 140201394498096
140193039418528 [label="encoder.layer.11.attention.self.key.weight
(768, 768)" fillcolor=lightblue]
140193039418528 -> 140201394499104
140201394499104 [label=AccumulateGrad]
140193036801264 -> 140193036801024
140193036801264 [label=UnsafeViewBackward0]
140193036801408 -> 140193036801264
140193036801408 [label=CloneBackward0]
140193036801600 -> 140193036801408
140193036801600 [label=ExpandBackward0]
140193036801792 -> 140193036801600
140193036801792 [label=PermuteBackward0]
140193036801360 -> 140193036801792
140193036801360 [label=ViewBackward0]
140193036802608 -> 140193036801360
140193036802608 [label=ViewBackward0]
140193036803280 -> 140193036802608
140193036803280 [label=AddmmBackward0]
140193036801216 -> 140193036803280
140193036801216 [label=ToCopyBackward0]
140201394498864 -> 140193036801216
140193039417888 [label="encoder.layer.11.attention.self.value.bias
(768)" fillcolor=lightblue]
140193039417888 -> 140201394498864
140201394498864 [label=AccumulateGrad]
140201394497136 -> 140193036803280
140201394497136 [label=ViewBackward0]
140201394499200 -> 140201394497136
140201394499200 [label=ToCopyBackward0]
140193036800208 -> 140201394499200
140201394497376 -> 140193036803280
140201394497376 [label=TBackward0]
140201394498768 -> 140201394497376
140201394498768 [label=ToCopyBackward0]
140201394499248 -> 140201394498768
140193039417808 [label="encoder.layer.11.attention.self.value.weight
(768, 768)" fillcolor=lightblue]
140193039417808 -> 140201394499248
140201394499248 [label=AccumulateGrad]
140193036800304 -> 140193036800496
140193036800304 [label=TBackward0]
140193036800976 -> 140193036800304
140193036800976 [label=ToCopyBackward0]
140193036801168 -> 140193036800976
140193039417568 [label="encoder.layer.11.attention.output.dense.weight
(768, 768)" fillcolor=lightblue]
140193039417568 -> 140193036801168
140193036801168 [label=AccumulateGrad]
140193036800208 -> 140193578430272
140193578430416 -> 140193578430176
140193039403264 [label="encoder.layer.11.attention.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039403264 -> 140193578430416
140193578430416 [label=AccumulateGrad]
140193578429696 -> 140193578430176
140193039404944 [label="encoder.layer.11.attention.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039404944 -> 140193578429696
140193578429696 [label=AccumulateGrad]
140193578428928 -> 140193578429216
140193578428928 [label=TBackward0]
140193578429936 -> 140193578428928
140193578429936 [label=ToCopyBackward0]
140193578430032 -> 140193578429936
140193039388080 [label="encoder.layer.11.experts.experts.0.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039388080 -> 140193578430032
140193578430032 [label=AccumulateGrad]
140193578428592 -> 140193578428880
140193578428592 [label=TBackward0]
140193578429360 -> 140193578428592
140193578429360 [label=ToCopyBackward0]
140193578429408 -> 140193578429360
140193039387760 [label="encoder.layer.11.experts.experts.0.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039387760 -> 140193578429408
140193578429408 [label=AccumulateGrad]
140193578428496 -> 140193578428256
140193578428496 [label=UnsqueezeBackward0]
140193578428640 -> 140193578428496
140193578428640 [label=NativeDropoutBackward0]
140193578429072 -> 140193578428640
140193578429072 [label=ViewBackward0]
140193578430224 -> 140193578429072
140193578430224 [label=AddmmBackward0]
140193578429264 -> 140193578430224
140193578429264 [label=ToCopyBackward0]
140193578430368 -> 140193578429264
140193039387840 [label="encoder.layer.11.experts.experts.1.dense2.bias
(768)" fillcolor=lightblue]
140193039387840 -> 140193578430368
140193578430368 [label=AccumulateGrad]
140193578429648 -> 140193578430224
140193578429648 [label=ViewBackward0]
140193578429600 -> 140193578429648
140193578429600 [label=GeluBackward0]
140193036800256 -> 140193578429600
140193036800256 [label=ViewBackward0]
140193036800688 -> 140193036800256
140193036800688 [label=AddmmBackward0]
140193036801312 -> 140193036800688
140193036801312 [label=ToCopyBackward0]
140193036801696 -> 140193036801312
140193039388240 [label="encoder.layer.11.experts.experts.1.dense1.bias
(3072)" fillcolor=lightblue]
140193039388240 -> 140193036801696
140193036801696 [label=AccumulateGrad]
140193036801072 -> 140193036800688
140193036801072 [label=ViewBackward0]
140193036801888 -> 140193036801072
140193036801888 [label=ToCopyBackward0]
140193578429840 -> 140193036801888
140193036800160 -> 140193036800688
140193036800160 [label=TBackward0]
140193036800880 -> 140193036800160
140193036800880 [label=ToCopyBackward0]
140193036801504 -> 140193036800880
140193039387520 [label="encoder.layer.11.experts.experts.1.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039387520 -> 140193036801504
140193036801504 [label=AccumulateGrad]
140193578428448 -> 140193578430224
140193578428448 [label=TBackward0]
140193578430320 -> 140193578428448
140193578430320 [label=ToCopyBackward0]
140193036802800 -> 140193578430320
140193039387280 [label="encoder.layer.11.experts.experts.1.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039387280 -> 140193036802800
140193036802800 [label=AccumulateGrad]
140193578426720 -> 140193578427056
140193578426720 [label=UnsqueezeBackward0]
140193578426768 -> 140193578426720
140193578426768 [label=UnsqueezeBackward0]
140193578428832 -> 140193578426768
140193578428832 [label=MulBackward0]
140193578430128 -> 140193578428832
140193578430128 [label=IndexBackward0]
140193578426576 -> 140193578430128
140193578426576 [label=SoftmaxBackward0]
140193036800112 -> 140193578426576
140193036800112 [label=CatBackward0]
140193036800448 -> 140193036800112
140193036800448 [label=MmBackward0]
140201394499296 -> 140193036800448
140201394499296 [label=MeanBackward1]
140193578428688 -> 140201394499296
140201394499344 -> 140193036800448
140201394499344 [label=TBackward0]
140201394499152 -> 140201394499344
140201394499152 [label=ToCopyBackward0]
140201394499536 -> 140201394499152
140193039401424 [label="encoder.layer.11.experts.gate.weight
(1, 768)" fillcolor=lightblue]
140193039401424 -> 140201394499536
140201394499536 [label=AccumulateGrad]
140201394499008 -> 140193036800112
140201394499008 [label=MmBackward0]
140201394499488 -> 140201394499008
140201394499488 [label=MeanBackward1]
140193578428640 -> 140201394499488
140201394499392 -> 140201394499008
140201394499392 [label=TBackward0]
140201394499152 -> 140201394499392
140193578427776 -> 140193578427584
140193578427776 [label=UnsqueezeBackward0]
140193578427008 -> 140193578427776
140193578427008 [label=SelectBackward0]
140193578427872 -> 140193578427008
140193578427872 [label=NativeDropoutBackward0]
140193578428544 -> 140193578427872
140193578428544 [label=ViewBackward0]
140193578429552 -> 140193578428544
140193578429552 [label=AddmmBackward0]
140193036802416 -> 140193578429552
140193036802416 [label=ToCopyBackward0]
140201394499440 -> 140193036802416
140193039387360 [label="encoder.layer.11.experts.experts.2.dense2.bias
(768)" fillcolor=lightblue]
140193039387360 -> 140201394499440
140201394499440 [label=AccumulateGrad]
140193036800544 -> 140193578429552
140193036800544 [label=ViewBackward0]
140201394548896 -> 140193036800544
140201394548896 [label=GeluBackward0]
140201394548992 -> 140201394548896
140201394548992 [label=ViewBackward0]
140201394549088 -> 140201394548992
140201394549088 [label=AddmmBackward0]
140201394549184 -> 140201394549088
140201394549184 [label=ToCopyBackward0]
140201394549376 -> 140201394549184
140193039387600 [label="encoder.layer.11.experts.experts.2.dense1.bias
(3072)" fillcolor=lightblue]
140193039387600 -> 140201394549376
140201394549376 [label=AccumulateGrad]
140201394549136 -> 140201394549088
140201394549136 [label=ViewBackward0]
140201394549424 -> 140201394549136
140201394549424 [label=ToCopyBackward0]
140201394549520 -> 140201394549424
140201394549520 [label=IndexBackward0]
140193578427392 -> 140201394549520
140201394548848 -> 140201394549088
140201394548848 [label=TBackward0]
140201394549616 -> 140201394548848
140201394549616 [label=ToCopyBackward0]
140201394549328 -> 140201394549616
140193039387040 [label="encoder.layer.11.experts.experts.2.dense1.weight
(3072, 768)" fillcolor=lightblue]
140193039387040 -> 140201394549328
140201394549328 [label=AccumulateGrad]
140201394499056 -> 140193578429552
140201394499056 [label=TBackward0]
140201394549040 -> 140201394499056
140201394549040 [label=ToCopyBackward0]
140201394549280 -> 140201394549040
140193039386800 [label="encoder.layer.11.experts.experts.2.dense2.weight
(768, 3072)" fillcolor=lightblue]
140193039386800 -> 140201394549280
140201394549280 [label=AccumulateGrad]
140193578427728 -> 140193578427632
140193578427728 [label=CatBackward0]
140193578426528 -> 140193578427728
140193578426528 [label=SliceBackward0]
140201394497856 -> 140193578426528
140201394497856 [label=SliceBackward0]
140193578428400 -> 140201394497856
140193578428400 [label=SliceBackward0]
140193578427152 -> 140193578428400
140193578427104 -> 140193578427728
140193578427104 [label=UnsqueezeBackward0]
140193578428112 -> 140193578427104
140193578428112 [label=SelectBackward0]
140193578427872 -> 140193578428112
140193578427488 -> 140193578427632
140193578427488 [label=CatBackward0]
140193578427920 -> 140193578427488
140193578427920 [label=SliceBackward0]
140201394548944 -> 140193578427920
140201394548944 [label=SliceBackward0]
140201394549472 -> 140201394548944
140201394549472 [label=SliceBackward0]
140193578427152 -> 140201394549472
140201394548800 -> 140193578427488
140201394548800 [label=UnsqueezeBackward0]
140201394549808 -> 140201394548800
140201394549808 [label=SelectBackward0]
140193578427872 -> 140201394549808
140193578427824 -> 140193578427632
140193578427824 [label=CatBackward0]
140201394549712 -> 140193578427824
140201394549712 [label=SliceBackward0]
140201394549856 -> 140201394549712
140201394549856 [label=SliceBackward0]
140201394549952 -> 140201394549856
140201394549952 [label=SliceBackward0]
140193578427152 -> 140201394549952
140201394549664 -> 140193578427824
140201394549664 [label=UnsqueezeBackward0]
140201394550048 -> 140201394549664
140201394550048 [label=SelectBackward0]
140193578427872 -> 140201394550048
140193578427392 -> 140193578427440
140193578427200 -> 140193578428064
140193039401344 [label="encoder.layer.11.expert_ln.weight
(768)" fillcolor=lightblue]
140193039401344 -> 140193578427200
140193578427200 [label=AccumulateGrad]
140193578427968 -> 140193578428064
140193039401104 [label="encoder.layer.11.expert_ln.bias
(768)" fillcolor=lightblue]
140193039401104 -> 140193578427968
140193578427968 [label=AccumulateGrad]
140193039092080 -> 140193570151248
140193039092080 [label=IndexBackward0]
140193578427536 -> 140193039092080
140193578427536 [label=IndexBackward0]
140193578428064 -> 140193578427536
140193039092320 -> 140193039136752
140193039092320 [label=AddBackward0]
140193578427680 -> 140193039092320
140193578427680 [label=IndexBackward0]
140193578428208 -> 140193578427680
140193578428208 [label=NativeLayerNormBackward0]
140201394550000 -> 140193578428208
140201394550000 [label=AddBackward0]
140201394550192 -> 140201394550000
140201394550192 [label=NativeDropoutBackward0]
140201394550336 -> 140201394550192
140201394550336 [label=ViewBackward0]
140201394550432 -> 140201394550336
140201394550432 [label=AddmmBackward0]
140201394550528 -> 140201394550432
140201394550528 [label=ToCopyBackward0]
140201394550720 -> 140201394550528
140193039403104 [label="encoder.layer.11.output.dense.bias
(768)" fillcolor=lightblue]
140193039403104 -> 140201394550720
140201394550720 [label=AccumulateGrad]
140201394550480 -> 140201394550432
140201394550480 [label=ViewBackward0]
140201394550768 -> 140201394550480
140201394550768 [label=GeluBackward0]
140201394550864 -> 140201394550768
140201394550864 [label=ViewBackward0]
140201394550960 -> 140201394550864
140201394550960 [label=AddmmBackward0]
140201394551056 -> 140201394550960
140201394551056 [label=ToCopyBackward0]
140201394551248 -> 140201394551056
140193039403344 [label="encoder.layer.11.intermediate.dense.bias
(3072)" fillcolor=lightblue]
140193039403344 -> 140201394551248
140201394551248 [label=AccumulateGrad]
140201394551008 -> 140201394550960
140201394551008 [label=ViewBackward0]
140201394551296 -> 140201394551008
140201394551296 [label=ToCopyBackward0]
140201394550144 -> 140201394551296
140201394550144 [label=SliceBackward0]
140201394551440 -> 140201394550144
140201394551440 [label=SliceBackward0]
140201394551536 -> 140201394551440
140201394551536 [label=SliceBackward0]
140193578430176 -> 140201394551536
140201394550672 -> 140201394550960
140201394550672 [label=TBackward0]
140201394551200 -> 140201394550672
140201394551200 [label=ToCopyBackward0]
140201394551632 -> 140201394551200
140193039403584 [label="encoder.layer.11.intermediate.dense.weight
(3072, 768)" fillcolor=lightblue]
140193039403584 -> 140201394551632
140201394551632 [label=AccumulateGrad]
140201394550240 -> 140201394550432
140201394550240 [label=TBackward0]
140201394550912 -> 140201394550240
140201394550912 [label=ToCopyBackward0]
140201394551392 -> 140201394550912
140193039403024 [label="encoder.layer.11.output.dense.weight
(768, 3072)" fillcolor=lightblue]
140193039403024 -> 140201394551392
140201394551392 [label=AccumulateGrad]
140201394550144 -> 140201394550000
140201394549760 -> 140193578428208
140193039402784 [label="encoder.layer.11.output.LayerNorm.weight
(768)" fillcolor=lightblue]
140193039402784 -> 140201394549760
140201394549760 [label=AccumulateGrad]
140201394549568 -> 140193578428208
140193039402864 [label="encoder.layer.11.output.LayerNorm.bias
(768)" fillcolor=lightblue]
140193039402864 -> 140201394549568
140201394549568 [label=AccumulateGrad]
140193578427296 -> 140193039092320
140193578427296 [label=IndexBackward0]
140201394550096 -> 140193578427296
140201394550096 [label=IndexBackward0]
140201394550384 -> 140201394550096
140201394550384 [label=IndexBackward0]
140193578428208 -> 140201394550384
140193039136752 -> 140193037219536
}
Reference in New Issue Copy Permalink