msluszniak commited on
Commit
c6ffc31
·
verified ·
1 Parent(s): 2579c1f

Add MLX int4 variant

Browse files
Files changed (3) hide show
  1. .gitattributes +1 -0
  2. mlx/config.json +61 -0
  3. mlx/phi_4_mini_mlx_int4.pte +3 -0
.gitattributes CHANGED
@@ -38,3 +38,4 @@ original/phi-4-mini_bf16.pte filter=lfs diff=lfs merge=lfs -text
38
  quantized/phi-4-mini_8da4w.pte filter=lfs diff=lfs merge=lfs -text
39
  xnnpack/phi_4_mini_xnnpack_bf16.pte filter=lfs diff=lfs merge=lfs -text
40
  xnnpack/phi_4_mini_xnnpack_8da4w.pte filter=lfs diff=lfs merge=lfs -text
 
 
38
  quantized/phi-4-mini_8da4w.pte filter=lfs diff=lfs merge=lfs -text
39
  xnnpack/phi_4_mini_xnnpack_bf16.pte filter=lfs diff=lfs merge=lfs -text
40
  xnnpack/phi_4_mini_xnnpack_8da4w.pte filter=lfs diff=lfs merge=lfs -text
41
+ mlx/phi_4_mini_mlx_int4.pte filter=lfs diff=lfs merge=lfs -text
mlx/config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "$schema": "https://huggingface.co/software-mansion/react-native-executorch-spec/resolve/main/config.schema.json",
3
+ "model": "phi_4_mini",
4
+ "family": "phi",
5
+ "capabilities": [
6
+ "text-generation"
7
+ ],
8
+ "backend": "mlx",
9
+ "license": "mit",
10
+ "variants": [
11
+ {
12
+ "file": "phi_4_mini_mlx_int4.pte",
13
+ "precision": "4w",
14
+ "quantized": true,
15
+ "default": true,
16
+ "methods": {
17
+ "enable_dynamic_shape": true,
18
+ "forward": {
19
+ "inputs": [
20
+ {
21
+ "shape": [
22
+ 1,
23
+ {
24
+ "min": 1,
25
+ "max": 2047
26
+ }
27
+ ],
28
+ "dtype": "int64"
29
+ },
30
+ {
31
+ "shape": [
32
+ 1
33
+ ],
34
+ "dtype": "int64"
35
+ }
36
+ ],
37
+ "outputs": [
38
+ {
39
+ "shape": [
40
+ 1,
41
+ 200064
42
+ ],
43
+ "dtype": "bfloat16"
44
+ }
45
+ ]
46
+ },
47
+ "get_bos_id": 199999,
48
+ "get_eos_ids": [
49
+ 200020,
50
+ 199999
51
+ ],
52
+ "get_max_context_len": 2048,
53
+ "get_max_seq_len": 2048,
54
+ "get_n_layers": 32,
55
+ "get_vocab_size": 200064,
56
+ "use_kv_cache": true,
57
+ "use_sdpa_with_kv_cache": true
58
+ }
59
+ }
60
+ ]
61
+ }
mlx/phi_4_mini_mlx_int4.pte ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:135d86947cab70e2e6f72a915afba1cd797baf5cf3d1babf591ad36c3bb0bd04
3
+ size 3267644544