From: Sondre Wold Date: Mon, 17 Jun 2024 11:48:26 +0000 (+0200) Subject: Test commit X-Git-Url: https://letsjmore.com/?a=commitdiff_plain;h=4b388f116d442a1d4a2d225ca650a10d3122a658;p=reference_transformer.git Test commit --- diff --git a/reference_transformer/tests/test_tokenizers.py b/reference_transformer/tests/test_tokenizers.py index f66c929..8e5a4d3 100644 --- a/reference_transformer/tests/test_tokenizers.py +++ b/reference_transformer/tests/test_tokenizers.py @@ -2,10 +2,9 @@ import pytest from reference_transformer.tokenizer import StupidTokenizer - @pytest.fixture def test_data(): - with open("./data/shakespeare.txt", 'r', encoding="utf8") as f: + with open("./data/shakespeare.txt", "r", encoding="utf8") as f: data = f.read() return data @@ -14,6 +13,4 @@ def test_stupid_encode_decode(test_data): tokenizer = StupidTokenizer(test_data) encoded = tokenizer.encode(test_data) decoded = tokenizer.decode(encoded) - assert test_data == decoded, "EncodeDecode does not reproduce original data..." - - + assert test_data == decoded, "EncodeDecode does not reproduce original data."