File size: 2,661 Bytes
d25442c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | import re
import string
import numpy as np
import torch
from torch import Tensor
from nltk.corpus import stopwords
stop_words = set(stopwords.words("russian"))
stop_words.difference_update({"не", "нет", "без"})
def data_preprocessing(text: str) -> str:
"""preprocessing string: lowercase, removing html-tags, punctuation,
stopwords, digits
Args:
text (str): input string for preprocessing
Returns:
str: preprocessed string
"""
text = text.lower()
text = re.sub("<.*?>", "", text) # html tags
text = "".join(
[c for c in text if c not in string.punctuation]
) # Remove punctuation
text = " ".join([word for word in text.split() if word not in stop_words])
text = " ".join([word for word in text.split() if not word.isdigit()])
return text
def get_words_by_freq(sorted_words: list[tuple[str, int]], n: int = 10) -> list:
return list(filter(lambda x: x[1] > n, sorted_words))
def padding(review_int: list, seq_len: int) -> np.array: # type: ignore
"""Make left-sided padding for input list of tokens
Args:
review_int (list): input list of tokens
seq_len (int): max length of sequence, it len(review_int[i]) > seq_len it will be trimmed, else it will be padded by zeros
Returns:
np.array: padded sequences
"""
features = np.zeros((len(review_int), seq_len), dtype=int)
for i, review in enumerate(review_int):
if len(review) <= seq_len:
zeros = list(np.zeros(seq_len - len(review)))
new = zeros + review
else:
new = review[:seq_len]
features[i, :] = np.array(new)
return features
def preprocess_single_string(
input_string: str, seq_len: int, vocab_to_int: dict, verbose: bool = False
) -> Tensor:
"""Function for all preprocessing steps on a single string
Args:
input_string (str): input single string for preprocessing
seq_len (int): max length of sequence, it len(review_int[i]) > seq_len it will be trimmed, else it will be padded by zeros
vocab_to_int (dict, optional): word corpus {'word' : int index}. Defaults to vocab_to_int.
Returns:
list: preprocessed string
"""
preprocessed_string = data_preprocessing(input_string)
result_list = []
for word in preprocessed_string.split():
try:
result_list.append(vocab_to_int[word])
except KeyError as e:
if verbose:
print(f"{e}: not in dictionary!")
pass
result_padded = padding([result_list], seq_len)[0]
return Tensor(result_padded)
|