File size: 3,993 Bytes
f00e721 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | __version__: str
from typing import TypedDict
class AssetInfo(TypedDict):
schema_version: int
morphology_sha256: str
model_sha256: str
runtime: str
python_hot_path: bool
compiled_surface_table_supported: bool
nedoformer_supported: bool
nedoformer_contract_version: int
nedoformer_lattice_schema_version: int
nedoformer_input_encoding_version: int
nedoformer_sidecar_supported: bool
nedoformer_sidecar_schema_version: int
class Tokenizer:
def __init__(
self,
mode: str = "auto",
max_sentence_tokens: int = 512,
max_fallback_chars: int = 48,
contextual_disambiguation: bool = True,
detect_unmarked_code: bool = True,
) -> None: ...
def tokenize_batch(self, documents: list[bytes], threads: int = 1) -> list[bytes]: ...
@staticmethod
def decode_batch(documents: list[bytes]) -> list[bytes]: ...
def roundtrip_batch(self, documents: list[bytes], threads: int = 1) -> bool: ...
class NedoFormerInputEncoding(TypedDict):
ids: list[int]
segment_offsets: list[int]
pooled_segments: list[int]
pool_spans: list[tuple[int, int]]
pool_modes: list[str]
pool_group_ids: list[int | None]
class NedoFormerTokenizer:
def __init__(
self,
mode: str = "auto",
max_sentence_tokens: int = 512,
max_fallback_chars: int = 48,
contextual_disambiguation: bool = True,
detect_unmarked_code: bool = True,
character_vocabulary: bytes | None = None,
generation_vocabulary: bytes | None = None,
compiled_analysis_table: bytes | None = None,
) -> None: ...
def lattice(self, document: bytes) -> bytes: ...
@staticmethod
def lattice_metadata_json(lattice: bytes) -> str: ...
def lattice_batch(self, documents: list[bytes], threads: int = 1) -> list[bytes]: ...
def lattice_sidecar(self, document: bytes) -> bytes: ...
def lattice_sidecar_batch(self, documents: list[bytes], threads: int = 1) -> list[bytes]: ...
def input_encoding(
self,
document: bytes,
policy: str = "best",
seed: int = 0,
temperature: float = 1.0,
) -> NedoFormerInputEncoding: ...
def input_encoding_from_sidecar(
self,
document: bytes,
sidecar: bytes,
policy: str = "best",
seed: int = 0,
temperature: float = 1.0,
) -> NedoFormerInputEncoding: ...
@staticmethod
def sample_lattice(
lattice: bytes,
policy: str = "best",
seed: int = 0,
temperature: float = 1.0,
) -> bytes: ...
def train_assets(
self,
documents: list[bytes],
max_chars: int = 500,
max_roots: int = 16000,
max_code_pieces: int = 4096,
) -> tuple[bytes, bytes, str]: ...
def generation_ids(self, document: bytes) -> list[int]: ...
def generation_ids_from_lattice(
self,
lattice: bytes,
policy: str = "best",
seed: int = 0,
temperature: float = 1.0,
) -> list[int]: ...
def generation_decode(self, ids: list[int]) -> bytes: ...
def contract_fingerprint(self) -> str: ...
class SurfaceTokenizer:
def __init__(
self,
vocabulary: bytes,
mode: str = "auto",
max_sentence_tokens: int = 512,
max_fallback_chars: int = 48,
contextual_disambiguation: bool = True,
detect_unmarked_code: bool = True,
analysis_table: bytes | None = None,
) -> None: ...
def inspect_json(self, document: bytes) -> str: ...
def encode_ids(self, document: bytes) -> list[int]: ...
def encode_ids_batch(self, documents: list[bytes], threads: int = 1) -> list[list[int]]: ...
def clear_runtime_caches(self) -> None: ...
def runtime_cache_stats(self, threads: int) -> dict[str, int]: ...
def decode_ids(self, ids: list[int]) -> bytes: ...
def vocabulary_size(self) -> int: ...
def asset_info() -> AssetInfo: ...
|