Hugging Face's logo

Asilarkness
/
Layer_generator

Model card Files
xet
 Community
Asilarkness commited on
Commit
02e05de
·
verified ·
1 Parent(s): a47b74c

Upload layers_found.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. layers_found.json +127 -109
layers_found.json CHANGED
@@ -3,115 +3,115 @@
3
  "conv": 1.0,
4
  "expr": 1.0,
5
  "speed": 0.3,
6
- "novelty": 0.5,
7
- "invalid": -2.0
8
  },
9
  "layers": [
10
  {
11
  "niche": [
12
- true,
13
  false,
14
  true,
15
- "L"
16
  ],
17
- "reward": -3.9022377252578737,
18
- "formula": "x + silu(sigmoid(dwconv_k5(relu(((scale(scale((sigmoid(x) + attn_h2((gelu(gelu(attn_h8(expc((gelu(shift(attn_h8(expc(((attn_h4(tanh(linear(((x * expc(neg(x))) * x)))) - x) + x))))) - x))))) - x))))) \u2299\u03c3 x) * x)))))",
19
  "metrics": {
20
- "early": 5.049893450737,
21
- "final": 3.6411195516586305,
22
- "step_ms": 9.455204010009766,
23
- "peak_gb": 0.740228608,
24
- "params": 4330496,
25
  "improved": true
26
  }
27
  },
28
  {
29
  "niche": [
30
- true,
31
- true,
32
- true,
33
- "M"
34
  ],
35
- "reward": -3.9450168907642365,
36
- "formula": "x + (neg(linear(dwconv_k5((relu(silu(tanh(expc(dwconv_k3(abs(linear(attn_h2(linear(expc(shift(neg(gelu(abs(relu(dwconv_k5(relu(tanh((silu((dwconv_k3(sigmoid(gelu(cummean(attn_h8(abs(expc(rmsnorm(shift(x))))))))) * x)) + x))))))))))))))))))) * x)))) * x)",
37
  "metrics": {
38
- "early": 4.384920120239258,
39
- "final": 3.4168367981910706,
40
- "step_ms": 7.356405258178711,
41
- "peak_gb": 0.691162112,
42
- "params": 3548672,
43
  "improved": true
44
  }
45
  },
46
  {
47
  "niche": [
 
48
  false,
49
  false,
50
- true,
51
- "M"
52
  ],
53
- "reward": -3.9851991057395932,
54
- "formula": "x + linear(dwconv_k5(x))",
55
  "metrics": {
56
- "early": 4.507479882240295,
57
- "final": 3.42824273109436,
58
- "step_ms": 2.8896331787109375,
59
- "peak_gb": 0.471178752,
60
- "params": 2232320,
61
  "improved": true
62
  }
63
  },
64
  {
65
  "niche": [
66
  true,
 
67
  true,
68
- true,
69
- "L"
70
  ],
71
- "reward": -4.005253511667251,
72
- "formula": "x + scale(linear(silu((dwconv_k3(((cummean((dwconv_k7(dwconv_k7(rmsnorm(cummean(gelu(abs(scale(attn_h2(attn_h2(silu(x)))))))))) * attn_h4(attn_h8((((gelu(abs(silu(scale(silu(attn_h8(attn_h8(shift(x)))))))) \u2299\u03c3 x) - x) * x))))) - x) * x)) + x))))",
73
  "metrics": {
74
- "early": 4.334499526023865,
75
- "final": 3.494627225399017,
76
- "step_ms": 15.115022659301758,
77
- "peak_gb": 0.779851264,
78
- "params": 5385728,
79
  "improved": true
80
  }
81
  },
82
  {
83
  "niche": [
84
- true,
85
  false,
86
  true,
87
- "M"
88
  ],
89
- "reward": -4.008926469087601,
90
- "formula": "x + neg(silu(tanh(((attn_h4(tanh((attn_h8(silu(dwconv_k5((gelu(dwconv_k7(tanh((rmsnorm(x) + gelu(dwconv_k5(tanh(scale(attn_h8(relu(dwconv_k7(shift(linear((neg(gelu(shift(neg((x \u2299\u03c3 x))))) * x)))))))))))))) - x)))) - x))) + x) \u2299\u03c3 x))))",
91
  "metrics": {
92
- "early": 4.365005278587342,
93
- "final": 3.5190891146659853,
94
- "step_ms": 11.14654541015625,
95
- "peak_gb": 0.690855424,
96
- "params": 3815424,
97
  "improved": true
98
  }
99
  },
100
  {
101
  "niche": [
102
- true,
103
  true,
104
  false,
105
- "L"
106
  ],
107
- "reward": -4.010219687223435,
108
- "formula": "x + neg((((relu(rmsnorm(attn_h4(silu((shift(scale(rmsnorm(shift((tanh(tanh(sigmoid(shift(cummean((tanh(relu(shift(attn_h4(tanh(scale(rmsnorm(attn_h8(cummean(attn_h8(abs(relu(shift(x))))))))))))) - x)))))) \u2299\u03c3 x))))) * x))))) \u2299\u03c3 x) - x) + x))",
109
  "metrics": {
110
- "early": 4.994764494895935,
111
- "final": 3.9244547486305237,
112
- "step_ms": 8.43501091003418,
113
- "peak_gb": 0.745585664,
114
- "params": 4196864,
115
  "improved": true
116
  }
117
  },
@@ -120,16 +120,16 @@
120
  false,
121
  false,
122
  false,
123
- "M"
124
  ],
125
- "reward": -4.157018375396729,
126
- "formula": "x + silu(linear(x))",
127
  "metrics": {
128
- "early": 4.364822912216186,
129
- "final": 3.9219311237335206,
130
- "step_ms": 2.2735595703125,
131
- "peak_gb": 0.47184384,
132
- "params": 2229760,
133
  "improved": true
134
  }
135
  },
@@ -137,35 +137,35 @@
137
  "niche": [
138
  true,
139
  false,
140
- false,
141
- "M"
142
  ],
143
- "reward": -4.63491507768631,
144
- "formula": "x + attn_h8(tanh(x))",
145
  "metrics": {
146
- "early": 5.195601058006287,
147
- "final": 4.034154748916626,
148
- "step_ms": 3.339529037475586,
149
- "peak_gb": 0.507380224,
150
- "params": 2622976,
151
  "improved": true
152
  }
153
  },
154
  {
155
  "niche": [
156
- true,
157
  false,
 
158
  false,
159
- "L"
160
  ],
161
- "reward": -4.664939749240875,
162
- "formula": "x + ((sigmoid(attn_h8((attn_h2(neg(attn_h8(attn_h2(linear(shift(neg(attn_h8(neg((neg(neg(neg((neg(attn_h2(neg(attn_h8(((linear(attn_h8((linear(attn_h8(attn_h8(x))) + x))) + x) - x))))) - x)))) - x)))))))))) - x))) + x) - x)",
163
  "metrics": {
164
- "early": 5.070250558853149,
165
- "final": 4.022561717033386,
166
- "step_ms": 19.75560188293457,
167
- "peak_gb": 0.772168192,
168
- "params": 7734784,
169
  "improved": true
170
  }
171
  },
@@ -174,34 +174,52 @@
174
  false,
175
  true,
176
  true,
177
- "M"
178
  ],
179
- "reward": -4.886961543560028,
180
- "formula": "x + (gelu(sigmoid((gelu(x) \u2299\u03c3 x))) \u2299\u03c3 shift(scale(cummean(dwconv_k7(x)))))",
181
  "metrics": {
182
- "early": 5.29410183429718,
183
- "final": 4.440862703323364,
184
- "step_ms": 3.2465457916259766,
185
- "peak_gb": 0.52446976,
186
- "params": 2102784,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
187
  "improved": true
188
  }
189
  },
190
  {
191
  "niche": [
192
- false,
193
  true,
194
  false,
195
- "M"
 
196
  ],
197
- "reward": -5.251126217842102,
198
- "formula": "x + expc(scale((sigmoid(sigmoid(x)) \u2299\u03c3 gelu(neg(neg(gelu(sigmoid(cummean(x)))))))))",
199
  "metrics": {
200
- "early": 5.5344810962677,
201
- "final": 4.923442649841308,
202
- "step_ms": 3.6940574645996094,
203
- "peak_gb": 0.530081792,
204
- "params": 2099200,
205
  "improved": true
206
  }
207
  },
@@ -210,16 +228,16 @@
210
  true,
211
  true,
212
  false,
213
- "M"
214
  ],
215
- "reward": -5.441527152061463,
216
- "formula": "x + (cummean(x) * attn_h8(neg(expc(attn_h4(x)))))",
217
  "metrics": {
218
- "early": 5.968507027626037,
219
- "final": 5.862407994270325,
220
- "step_ms": 4.344940185546875,
221
- "peak_gb": 0.554299904,
222
- "params": 3147264,
223
  "improved": true
224
  }
225
  }
 
3
  "conv": 1.0,
4
  "expr": 1.0,
5
  "speed": 0.3,
6
+ "novelty": 1.0,
7
+ "invalid": -1.0
8
  },
9
  "layers": [
10
  {
11
  "niche": [
12
+ false,
13
  false,
14
  true,
15
+ true
16
  ],
17
+ "reward": 5.821100126778342,
18
+ "formula": "x + gelu(glu2(gelu(neg(tanh(gelu(dwconv_k3(relu(linear(gelu(gelu(gelu(rmsnorm(x)))))))))))))",
19
  "metrics": {
20
+ "early": 4.507159852981568,
21
+ "final": 3.2640185594558715,
22
+ "step_ms": 50.704002380371094,
23
+ "peak_gb": 11.556068864,
24
+ "params": 6248832,
25
  "improved": true
26
  }
27
  },
28
  {
29
  "niche": [
30
+ false,
31
+ false,
32
+ false,
33
+ true
34
  ],
35
+ "reward": 5.745956145798423,
36
+ "formula": "x + mlp2(silu(x))",
37
  "metrics": {
38
+ "early": 4.346919012069702,
39
+ "final": 3.783213210105896,
40
+ "step_ms": 33.315181732177734,
41
+ "peak_gb": 8.055067648,
42
+ "params": 4918272,
43
  "improved": true
44
  }
45
  },
46
  {
47
  "niche": [
48
+ true,
49
  false,
50
  false,
51
+ false
 
52
  ],
53
+ "reward": 5.486117064034202,
54
+ "formula": "x + gelu((gelu((x \u2299\u03c3 (gelu((x \u2299\u03c3 scale(x))) \u2299\u03c3 scale(x)))) \u2299\u03c3 attn_h8(x)))",
55
  "metrics": {
56
+ "early": 4.561727523803711,
57
+ "final": 3.775619101524353,
58
+ "step_ms": 59.35382843017578,
59
+ "peak_gb": 11.285428736,
60
+ "params": 4920576,
61
  "improved": true
62
  }
63
  },
64
  {
65
  "niche": [
66
  true,
67
+ false,
68
  true,
69
+ false
 
70
  ],
71
+ "reward": 5.154641150509574,
72
+ "formula": "x + gelu(gelu(gelu((x \u2299\u03c3 scale(relu(linear(gelu(dwconv_k3(gelu(gelu((scale(relu(linear(gelu(x)))) \u2299\u03c3 scale(relu(linear(gelu((scale(relu(linear(gelu((scale(relu(linear(gelu((attn_h8(linear(gelu((scale(relu(linear(gelu(dwconv_k3(x))))) \u2299\u03c3 scale(x))))) \u2299\u03c3 x))))) \u2299\u03c3 x))))) \u2299\u03c3 x)))))))))))))))))",
73
  "metrics": {
74
+ "early": 4.642396020889282,
75
+ "final": 3.6228970527648925,
76
+ "step_ms": 120.60427665710449,
77
+ "peak_gb": 17.224617984,
78
+ "params": 8029824,
79
  "improved": true
80
  }
81
  },
82
  {
83
  "niche": [
84
+ false,
85
  false,
86
  true,
87
+ false
88
  ],
89
+ "reward": 5.121382938896873,
90
+ "formula": "x + gelu(gelu((scale(relu(linear(gelu((scale(relu(linear(x))) \u2299\u03c3 scale(relu(linear(gelu(silu(x)))))))))) \u2299\u03c3 scale(relu(linear(gelu(neg(dwconv_k3(linear(x))))))))))",
91
  "metrics": {
92
+ "early": 3.988122344017029,
93
+ "final": 3.1123839378356934,
94
+ "step_ms": 56.54621124267578,
95
+ "peak_gb": 11.528669696,
96
+ "params": 5368704,
97
  "improved": true
98
  }
99
  },
100
  {
101
  "niche": [
102
+ false,
103
  true,
104
  false,
105
+ true
106
  ],
107
+ "reward": 4.835630094086387,
108
+ "formula": "x + expc(neg(gelu(gelu((scale(relu(linear(sigmoid(mlp4(gelu(linear(gelu(neg(tanh(tanh(scale(relu(linear(gelu(rmsnorm(x)))))))))))))))) \u2299\u03c3 scale(relu(linear(cummean(scale(relu(neg((gelu(glu2(gelu(tanh(scale(relu(linear(gelu(x)))))))) - scale(relu(linear(gelu((scale(relu(x)) \u2299\u03c3 x))))))))))))))))))",
109
  "metrics": {
110
+ "early": 5.217595434188842,
111
+ "final": 3.9673903226852416,
112
+ "step_ms": 97.13172912597656,
113
+ "peak_gb": 17.459894272,
114
+ "params": 12004224,
115
  "improved": true
116
  }
117
  },
 
120
  false,
121
  false,
122
  false,
123
+ false
124
  ],
125
+ "reward": 4.823786722695091,
126
+ "formula": "x + gelu((scale(relu(gelu(neg(tanh(x))))) \u2299\u03c3 scale(relu(rmsnorm(x)))))",
127
  "metrics": {
128
+ "early": 4.0697242498397825,
129
+ "final": 3.7892616271972654,
130
+ "step_ms": 42.938947677612305,
131
+ "peak_gb": 9.84554752,
132
+ "params": 3151104,
133
  "improved": true
134
  }
135
  },
 
137
  "niche": [
138
  true,
139
  false,
140
+ true,
141
+ true
142
  ],
143
+ "reward": 4.598309086834647,
144
+ "formula": "x + gelu((scale(relu(linear(x))) \u2299\u03c3 scale(relu(linear(gelu(gelu((x \u2299\u03c3 mlp4(attn_h8((x - scale(relu(abs(relu(linear(gelu((dwconv_k3(silu(x)) \u2299\u03c3 scale(relu(linear(gelu(abs(dwconv_k7(rmsnorm(x)))))))))))))))))))))))))",
145
  "metrics": {
146
+ "early": 3.9827035903930663,
147
+ "final": 3.7270321369171144,
148
+ "step_ms": 92.9560661315918,
149
+ "peak_gb": 15.793202688,
150
+ "params": 10242816,
151
  "improved": true
152
  }
153
  },
154
  {
155
  "niche": [
 
156
  false,
157
+ true,
158
  false,
159
+ false
160
  ],
161
+ "reward": 4.508914195572593,
162
+ "formula": "x + expc((relu(linear(gelu((scale(relu(linear(gelu(neg(gelu(x)))))) \u2299\u03c3 scale(relu(linear(gelu(cummean(scale(relu(linear(gelu((scale(relu(linear(gelu(cummean(gelu(((scale(relu(linear(gelu(gelu((scale(relu(linear(gelu(gelu(x))))) \u2299\u03c3 x)))))) \u2299\u03c3 x) + x))))))) \u2299\u03c3 x)))))))))))))) - x))",
163
  "metrics": {
164
+ "early": 4.0373770236969,
165
+ "final": 3.7183741331100464,
166
+ "step_ms": 104.02059555053711,
167
+ "peak_gb": 16.417043968,
168
+ "params": 6252288,
169
  "improved": true
170
  }
171
  },
 
174
  false,
175
  true,
176
  true,
177
+ true
178
  ],
179
+ "reward": 4.334455405747153,
180
+ "formula": "x + gelu((scale(relu(linear(gelu((scale(relu(linear(dwconv_k3(relu(gelu(dwconv_k3(relu(linear(gelu(neg(cummean(relu(linear(gelu(linear(gelu(linear(silu(dwconv_k7(gelu(rmsnorm(x)))))))))))))))))))))) \u2299\u03c3 glu2(rmsnorm(x))))))) \u2299\u03c3 scale(relu(linear(gelu(rmsnorm(x)))))))",
181
  "metrics": {
182
+ "early": 4.589245700836182,
183
+ "final": 3.7722170114517213,
184
+ "step_ms": 82.62109756469727,
185
+ "peak_gb": 14.808715264,
186
+ "params": 8918016,
187
+ "improved": true
188
+ }
189
+ },
190
+ {
191
+ "niche": [
192
+ true,
193
+ true,
194
+ true,
195
+ true
196
+ ],
197
+ "reward": 4.187065421616294,
198
+ "formula": "x + gelu(mlp4(scale(gelu((dwconv_k3(((linear(gelu(scale(dwconv_k3(rmsnorm(dwconv_k7(dwconv_k7(gelu(dwconv_k3(relu(rmsnorm(mlp4(linear(x))))))))))))) \u2299\u03c3 scale(relu(neg(cummean(attn_h8(tanh(attn_h8(tanh(tanh(abs(glu2(glu2(neg(dwconv_k7((rmsnorm(neg(neg(relu(dwconv_k5(attn_h2(scale(x))))))) + x)))))))))))))))) \u2299\u03c3 x)) + x)))))",
199
+ "metrics": {
200
+ "early": 4.385487127304077,
201
+ "final": 3.1699397802352904,
202
+ "step_ms": 174.35574531555176,
203
+ "peak_gb": 22.777659904,
204
+ "params": 21773184,
205
  "improved": true
206
  }
207
  },
208
  {
209
  "niche": [
 
210
  true,
211
  false,
212
+ false,
213
+ true
214
  ],
215
+ "reward": 4.0684950223319305,
216
+ "formula": "x + gelu((gelu((scale(relu(linear(gelu((scale(relu(linear(gelu(attn_h4(x))))) \u2299\u03c3 scale(x)))))) \u2299\u03c3 (scale(relu(linear(gelu(glu2(gelu(x)))))) \u2299\u03c3 scale(relu(linear(gelu((scale(relu(linear(gelu((x \u2299\u03c3 scale(relu(linear(gelu((scale(relu(linear(x))) \u2299\u03c3 x)))))))))) \u2299\u03c3 x)))))))) \u2299\u03c3 x))",
217
  "metrics": {
218
+ "early": 4.622024250030518,
219
+ "final": 3.7339990377426147,
220
+ "step_ms": 127.40111351013184,
221
+ "peak_gb": 19.977999872,
222
+ "params": 10678272,
223
  "improved": true
224
  }
225
  },
 
228
  true,
229
  true,
230
  false,
231
+ false
232
  ],
233
+ "reward": 3.7166457643859165,
234
+ "formula": "x + cummean(relu(gelu(scale(relu(attn_h4(x))))))",
235
  "metrics": {
236
+ "early": 5.998728942871094,
237
+ "final": 5.8545290470123295,
238
+ "step_ms": 61.27309799194336,
239
+ "peak_gb": 9.077064704,
240
+ "params": 4919424,
241
  "improved": true
242
  }
243
  }