update: look at todo

main
Thamognya Kodi 2023-01-01 20:01:12 +07:00
parent 7da9bad2af
commit c541ed92e5
11 changed files with 2725 additions and 62 deletions

4
.github/TODO.md vendored
View File

@ -1,2 +1,4 @@
- Get package to use .env from django - [ ] make frontend
- [ ] find better way of using .env from django (possibly)
- [x] Get package to use .env from django
- [x] IMP IMP work on API_KEY fixing config or use .env - [x] IMP IMP work on API_KEY fixing config or use .env

View File

@ -123,9 +123,17 @@ pytestcache-remove:
pytestcache-remove: pytestcache-remove:
find . | grep -E ".log" | xargs rm -rf find . | grep -E ".log" | xargs rm -rf
.PHONY: env-remove
pytestcache-remove:
find . | grep -E ".env" | xargs rm -rf
.PHONY: env-fishremove
pytestcache-remove:
find . | grep -E ".env-fish" | xargs rm -rf
.PHONY: build-remove .PHONY: build-remove
build-remove: build-remove:
rm -rf build/ rm -rf build/
.PHONY: cleanup .PHONY: cleanup
cleanup: pycache-remove dsstore-remove mypycache-remove ipynbcheckpoints-remove pytestcache-remove pkl-remove log-remove cleanup: pycache-remove dsstore-remove mypycache-remove ipynbcheckpoints-remove pytestcache-remove pkl-remove log-remove env-remove env-fishremove

View File

@ -8,8 +8,6 @@ from pathlib import Path
import dotenv import dotenv
from transformers import pipeline from transformers import pipeline
dotenv.load_dotenv()
logging.basicConfig( logging.basicConfig(
filename="QA.log", filename="QA.log",
filemode="w", filemode="w",
@ -27,8 +25,6 @@ def answer(
query: str, GOOGLE_SEARCH_API_KEY: str, GOOGLE_SEARCH_ENGINE_ID: str query: str, GOOGLE_SEARCH_API_KEY: str, GOOGLE_SEARCH_ENGINE_ID: str
) -> tuple[Any, list[str]]: ) -> tuple[Any, list[str]]:
QA_MODEL: Any = pipeline("question-answering") QA_MODEL: Any = pipeline("question-answering")
# GOOGLE_SEARCH_API_KEY = str(os.getenv("INTERNET_ML_GOOGLE_API"))
# GOOGLE_SEARCH_ENGINE_ID = str(os.getenv("INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID"))
results: tuple[list[str], list[str]] = internet.Google( results: tuple[list[str], list[str]] = internet.Google(
query, GOOGLE_SEARCH_API_KEY, GOOGLE_SEARCH_ENGINE_ID query, GOOGLE_SEARCH_API_KEY, GOOGLE_SEARCH_ENGINE_ID
).google() ).google()
@ -41,6 +37,6 @@ def answer(
return answer return answer
# print(answer("Who is the author of TinTin?")) # print(os.environ)
# print(answer("Who is Elon Musk?"))
# def custom_answer # def custom_answer

View File

@ -9,11 +9,6 @@ from pathlib import Path
import dotenv import dotenv
import requests import requests
dotenv.load_dotenv()
# GOOGLE_SEARCH_API_KEY = str(os.environ["INTERNET_ML_GOOGLE_API"])
# GOOGLE_SEARCH_ENGINE_ID = str(os.environ["INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID"])
HTTP_USERAGENT: dict[str, str] = { HTTP_USERAGENT: dict[str, str] = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
} }
@ -54,6 +49,20 @@ class Google:
) -> None: ) -> None:
self.__GOOGLE_SEARCH_API_KEY: str = GOOGLE_SEARCH_API_KEY self.__GOOGLE_SEARCH_API_KEY: str = GOOGLE_SEARCH_API_KEY
self.__GOOGLE_SEARCH_ENGINE_ID: str = GOOGLE_SEARCH_ENGINE_ID self.__GOOGLE_SEARCH_ENGINE_ID: str = GOOGLE_SEARCH_ENGINE_ID
# dotenv_path = os.path.join(os.path.dirname(__file__), '.env')
# dotenv.load_dotenv(dotenv_path)
# self.__GOOGLE_SEARCH_API_KEY: str = ""
# self.__GOOGLE_SEARCH_ENGINE_ID: str = ""
# if (
# "INTERNET_ML_GOOGLE_API" in os.environ
# and "INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID" in os.environ
# ):
# self.__GOOGLE_SEARCH_API_KEY = str(os.environ.get("INTERNET_ML_GOOGLE_API"))
# self.__GOOGLE_SEARCH_ENGINE_ID = str(
# os.environ.get("INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID")
# )
# else:
# exit("API KEYS")
self.__num_res: int = ( self.__num_res: int = (
5 5
if config.NLP_CONF_MODE == "speed" if config.NLP_CONF_MODE == "speed"
@ -62,8 +71,12 @@ class Google:
self.__query = query self.__query = query
self.__URL_EXTRACTOR: URLExtract = URLExtract() self.__URL_EXTRACTOR: URLExtract = URLExtract()
self.__urls: list[str] = self.__URL_EXTRACTOR.find_urls(query) self.__urls: list[str] = self.__URL_EXTRACTOR.find_urls(query)
self.__query = re.sub( self.__query = str(
r"\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*", "", self.__query re.sub(
r"\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*",
"",
str(self.__query),
)
) )
def __get_urls(self: "Google") -> None: def __get_urls(self: "Google") -> None:
@ -152,10 +165,6 @@ class Google:
return (self.__content, self.__urls) return (self.__content, self.__urls)
# def google(query: str) -> tuple[list[str], list[str]]:
# return Google(query).google()
""" """
Timing: Timing:
import time import time

View File

@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry] [tool.poetry]
name = "internet-ml" name = "internet-ml"
version = "1.0.6" version = "1.0.20"
description = "Internet-ML: Allowing ML to connect to the internet" description = "Internet-ML: Allowing ML to connect to the internet"
readme = "./.github/README.md" readme = "./.github/README.md"
authors = ["Thamognya Kodi <contact@thamognya.com>"] authors = ["Thamognya Kodi <contact@thamognya.com>"]

View File

@ -1,3 +0,0 @@
from django.test import TestCase
# Create your tests here.

View File

@ -1,25 +0,0 @@
from typing import Any
import json
import internet_ml.NLP.no_context.QA
def QA(
query: str, INTERNET_ML_GOOGLE_API: str, INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID: str
) -> Any:
try:
answer = internet_ml.NLP.no_context.QA.answer(
query, INTERNET_ML_GOOGLE_API, INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID
)
content = json.dumps(
{"error": "", "response": answer[0], "resources": answer[1]}
)
except:
content = json.dumps(
{"error": "Google API key not present in .env or environment"}
)
return content
# print(QA("Who is Elon Musk?"))

View File

@ -1,19 +1,18 @@
import json
import os import os
from dotenv import load_dotenv import dotenv
from rest_framework import status from rest_framework import status
from rest_framework.response import Response from rest_framework.response import Response
from rest_framework.views import APIView from rest_framework.views import APIView
from .tools import question_answer
load_dotenv()
import internet_ml.NLP.no_context.QA import internet_ml.NLP.no_context.QA
dotenv_path = os.path.join(os.path.dirname(__file__), ".env")
dotenv.load_dotenv(dotenv_path)
class QAView(APIView): class QAView(APIView):
def post(self, request, format=None): def post(self, request):
""" """
{"question": "Who is Elon Musk?"} {"question": "Who is Elon Musk?"}
{ {
@ -37,11 +36,14 @@ class QAView(APIView):
so check error if it exists first and then for other stuff so check error if it exists first and then for other stuff
""" """
answer = internet_ml.NLP.no_context.QA.answer( answer = internet_ml.NLP.no_context.QA.answer(
request.POST.get("question"), str(request.data["question"]),
str(os.getenv("INTERNET_ML_GOOGLE_API")), str(os.environ.get("INTERNET_ML_GOOGLE_API")),
str(os.getenv("INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID")), str(os.environ.get("INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID")),
) )
content = json.dumps( content = {
{"error": "", "response": answer[0], "resources": answer[1]} "error": "",
) "question": str(request.data["question"]),
return content "response": answer[0],
"resources": answer[1],
}
return Response(content)

File diff suppressed because it is too large Load Diff

View File

@ -17,6 +17,7 @@ django-stubs = "^1.13.1"
djangorestframework = "^3.14.0" djangorestframework = "^3.14.0"
python-dotenv = "^0.21.0" python-dotenv = "^0.21.0"
django-environ = "^0.9.0" django-environ = "^0.9.0"
internet-ml = "^1.0.11"
[tool.mypy] [tool.mypy]
plugins = ["mypy_django_plugin.main"] plugins = ["mypy_django_plugin.main"]