thamognya
/
internet_ml
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update: look at todo

main
Thamognya Kodi 2022-12-30 12:28:26 +07:00
parent 945a41aefe
commit 71e7a3c3ec
2 changed files with 20 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
internet_ml/NLP/no_context/QA.py
View File

@ -1,11 +1,15 @@
from typing import Any, List, Tuple from typing import Any, List, Tuple
import logging import logging
import os
import sys import sys
from pathlib import Path from pathlib import Path
import dotenv
from transformers import pipeline from transformers import pipeline
dotenv.load_dotenv()
logging.basicConfig( logging.basicConfig(
filename="QA.log", filename="QA.log",
filemode="w", filemode="w",
@ -18,12 +22,17 @@ sys.path.append(str(Path(__file__).parent.parent.parent) + "/utils")
import config import config
import internet import internet
GOOGLE_SEARCH_API_KEY = str(os.environ["INTERNET_ML_GOOGLE_API"])
GOOGLE_SEARCH_ENGINE_ID = str(os.environ["INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID"])
QA_MODEL: Any = pipeline("question-answering") QA_MODEL: Any = pipeline("question-answering")
def answer(query: str) -> tuple[Any, list[str]]: def answer(query: str) -> tuple[Any, list[str]]:
global QA_MODEL global QA_MODEL
results: tuple[list[str], list[str]] = internet.google(query) results: tuple[list[str], list[str]] = internet.Google(
query, GOOGLE_SEARCH_API_KEY, GOOGLE_SEARCH_ENGINE_ID
).google()
answer: tuple[Any, list[str]] = ( answer: tuple[Any, list[str]] = (
QA_MODEL(question=query, context=str(results[0])), QA_MODEL(question=query, context=str(results[0])),
results[1], results[1],

15
internet_ml/tools/NLP/data/internet.py
View File

@ -11,8 +11,8 @@ import requests
dotenv.load_dotenv() dotenv.load_dotenv()
GOOGLE_SEARCH_API_KEY = str(os.environ["INTERNET_ML_GOOGLE_API"]) # GOOGLE_SEARCH_API_KEY = str(os.environ["INTERNET_ML_GOOGLE_API"])
GOOGLE_SEARCH_ENGINE_ID = str(os.environ["INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID"]) # GOOGLE_SEARCH_ENGINE_ID = str(os.environ["INTERNET_ML_GOOGLE_SEARCH_ENGINE_ID"])
HTTP_USERAGENT: dict[str, str] = { HTTP_USERAGENT: dict[str, str] = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
@ -46,7 +46,12 @@ from urlextract import URLExtract
class Google: class Google:
def __init__(self: "Google", query: str) -> None: def __init__(
self: "Google",
query: str,
GOOGLE_SEARCH_API_KEY: str,
GOOGLE_SEARCH_ENGINE_ID: str,
) -> None:
self.__GOOGLE_SEARCH_API_KEY: str = GOOGLE_SEARCH_API_KEY self.__GOOGLE_SEARCH_API_KEY: str = GOOGLE_SEARCH_API_KEY
self.__GOOGLE_SEARCH_ENGINE_ID: str = GOOGLE_SEARCH_ENGINE_ID self.__GOOGLE_SEARCH_ENGINE_ID: str = GOOGLE_SEARCH_ENGINE_ID
self.__num_res: int = ( self.__num_res: int = (
@ -147,8 +152,8 @@ class Google:
return (self.__content, self.__urls) return (self.__content, self.__urls)
def google(query: str) -> tuple[list[str], list[str]]: # def google(query: str) -> tuple[list[str], list[str]]:
return Google(query).google() # return Google(query).google()
""" """