11. NLP Cheatsheet

Tokenizer

python
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
tokens = tokenizer("Hello world", return_tensors="pt")

python
import re
# Find emails
re.findall(r'[\w\.-]+@[\w\.-]+', text)

python
model = SentenceTransformer('all-MiniLM-L6-v2')
emb = model.encode("Hello world")