update: look at todo
parent
5f0faa77a4
commit
5261d734de
|
|
@ -1,4 +1,13 @@
|
|||
# type: ignore
|
||||
|
||||
"""
|
||||
model naming convention
|
||||
# Open-AI models:
|
||||
include prefix openai-*
|
||||
# HuggingFace
|
||||
include prefix hf-*
|
||||
"""
|
||||
|
||||
from typing import Any, List, Tuple
|
||||
|
||||
import os
|
||||
|
|
@ -31,11 +40,6 @@ def answer(
|
|||
CHATGPT_CONVERSATION_ID: str = "",
|
||||
CHATGPT_PARENT_ID: str = "",
|
||||
) -> tuple[Any, list[str]]:
|
||||
# if environment keys are not given, assume it is in env
|
||||
if GOOGLE_SEARCH_API_KEY == "":
|
||||
GOOGLE_SEARCH_API_KEY = str(os.environ.get("GOOGLE_SEARCH_API_KEY"))
|
||||
if GOOGLE_SEARCH_ENGINE_ID == "":
|
||||
GOOGLE_SEARCH_ENGINE_ID = str(os.environ.get("GOOGLE_SEARCH_ENGINE_ID"))
|
||||
if OPENAI_API_KEY == "":
|
||||
OPENAI_API_KEY = str(os.environ.get("OPENAI_API_KEY"))
|
||||
openai.api_key = OPENAI_API_KEY
|
||||
|
|
@ -45,25 +49,20 @@ def answer(
|
|||
CHATGPT_CONVERSATION_ID = str(os.environ.get("CHATGPT_CONVERSATION_ID"))
|
||||
if CHATGPT_PARENT_ID == "":
|
||||
CHATGPT_PARENT_ID = str(os.environ.get("CHATGPT_PARENT_ID"))
|
||||
"""
|
||||
model naming convention
|
||||
# Open-AI models:
|
||||
include prefix openai-*
|
||||
# HuggingFace
|
||||
include prefix hf-*
|
||||
#
|
||||
"""
|
||||
|
||||
if not (model.startswith("openai-") or model.startswith("hf-")):
|
||||
model = "openai-chatgpt" # Default
|
||||
if model.startswith("openai-"):
|
||||
if model == "openai-chatgpt":
|
||||
# ChatGPT
|
||||
|
||||
results: tuple[list[str], list[str]] = internet.Google(
|
||||
query, GOOGLE_SEARCH_API_KEY, GOOGLE_SEARCH_ENGINE_ID
|
||||
).google()
|
||||
# print(' '.join(filter(lambda x: isinstance(x, str), results[0]))[:4000])
|
||||
|
||||
if model.startswith("openai-"):
|
||||
if model == "openai-chatgpt":
|
||||
# ChatGPT
|
||||
prompt = f"Using the context: {' '.join(filter(lambda x: isinstance(x, str), results[0]))[:3000]} and answer the question with the context above and previous knowledge: \"{query}\". Also write long answers or essays if asked."
|
||||
print(prompt)
|
||||
exit(1)
|
||||
chatbot = Chatbot(
|
||||
{"session_token": CHATGPT_SESSION_TOKEN},
|
||||
conversation_id=None,
|
||||
|
|
@ -77,14 +76,11 @@ def answer(
|
|||
return (response["message"], results[1])
|
||||
else:
|
||||
if model == "openai-text-davinci-003":
|
||||
results: tuple[list[str], list[str]] = internet.Google(
|
||||
query, GOOGLE_SEARCH_API_KEY, GOOGLE_SEARCH_ENGINE_ID
|
||||
).google()
|
||||
context = " ".join(results[0])
|
||||
context[: (4097 - len(query) - 10)]
|
||||
# text-davinci-003
|
||||
prompt = f"Using the context: {' '.join(filter(lambda x: isinstance(x, str), results[0]))[:3000]} and answer the question with the context above and previous knowledge: \"{query}\". Also write long answers or essays if asked."
|
||||
response = openai.Completion.create(
|
||||
model="text-davinci-003",
|
||||
prompt=f"{context} Q: {query}",
|
||||
prompt=prompt,
|
||||
max_tokens=len(context),
|
||||
n=1,
|
||||
stop=None,
|
||||
|
|
@ -94,9 +90,6 @@ def answer(
|
|||
# TODO: add suport later
|
||||
else:
|
||||
model = model.replace("hf-", "", 1)
|
||||
results: tuple[list[str], list[str]] = internet.Google(
|
||||
query, GOOGLE_SEARCH_API_KEY, GOOGLE_SEARCH_ENGINE_ID
|
||||
).google()
|
||||
qa_model = pipeline("question-answering", model=model)
|
||||
response = qa_model(question=query, context=" ".join(results[0]))
|
||||
return (response["answer"], results[1])
|
||||
|
|
|
|||
|
|
@ -1,6 +1,142 @@
|
|||
from typing import Any, List, Tuple
|
||||
# from typing import Any, List, Tuple
|
||||
|
||||
# import os
|
||||
# import sys
|
||||
# from importlib import reload
|
||||
# from pathlib import Path
|
||||
|
||||
# import dotenv
|
||||
# import requests
|
||||
|
||||
# HTTP_USERAGENT: dict[str, str] = {
|
||||
# "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
|
||||
# }
|
||||
|
||||
# sys.path.append(str(Path(__file__).parent.parent.parent.parent) + "/utils/NLP")
|
||||
# sys.path.append(str(Path(__file__).parent.parent.parent.parent) + "/utils")
|
||||
# sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
# import asyncio
|
||||
# import itertools
|
||||
# import re
|
||||
|
||||
# import aiohttp
|
||||
# import config
|
||||
# from bs4 import BeautifulSoup
|
||||
# from normalize import normalizer
|
||||
|
||||
# # from relevancy import filter_irrelevant
|
||||
# from sentencize import sentencizer
|
||||
# from urlextract import URLExtract
|
||||
# from adremover import AdRemover
|
||||
|
||||
|
||||
# class Google:
|
||||
# def __init__(
|
||||
# self: "Google",
|
||||
# query: str,
|
||||
# GOOGLE_SEARCH_API_KEY: str,
|
||||
# GOOGLE_SEARCH_ENGINE_ID: str,
|
||||
# ) -> None:
|
||||
# self.__GOOGLE_SEARCH_API_KEY: str = GOOGLE_SEARCH_API_KEY
|
||||
# self.__GOOGLE_SEARCH_ENGINE_ID: str = GOOGLE_SEARCH_ENGINE_ID
|
||||
|
||||
# # if environment keys are not given, assume it is in env
|
||||
# if GOOGLE_SEARCH_API_KEY == "":
|
||||
# self.__GOOGLE_SEARCH_API_KEY = str(os.environ.get("GOOGLE_SEARCH_API_KEY"))
|
||||
# if GOOGLE_SEARCH_ENGINE_ID == "":
|
||||
# self.__GOOGLE_SEARCH_ENGINE_ID = str(os.environ.get("GOOGLE_SEARCH_ENGINE_ID"))
|
||||
|
||||
# self.__num_res: int = 10
|
||||
# self.__query = query
|
||||
# self.__URL_EXTRACTOR: URLExtract = URLExtract()
|
||||
# self.__urls: list[str] = self.__URL_EXTRACTOR.find_urls(query)
|
||||
# self.__query = str(
|
||||
# re.sub(
|
||||
# r"\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*",
|
||||
# "",
|
||||
# str(self.__query),
|
||||
# )
|
||||
# )
|
||||
|
||||
# def __get_urls(self: "Google") -> None:
|
||||
# if self.__GOOGLE_SEARCH_API_KEY == "":
|
||||
# exit("ERROR: Google API Key not found")
|
||||
# if self.__GOOGLE_SEARCH_ENGINE_ID == "":
|
||||
# exit("ERROR: Google Search Engine Id not found")
|
||||
# response = requests.get(
|
||||
# "https://www.googleapis.com/customsearch/v1",
|
||||
# params={
|
||||
# "key": self.__GOOGLE_SEARCH_API_KEY,
|
||||
# "q": self.__query,
|
||||
# "cx": self.__GOOGLE_SEARCH_ENGINE_ID,
|
||||
# },
|
||||
# )
|
||||
# results = response.json()["items"]
|
||||
# for result in results:
|
||||
# self.__urls.append(result["link"])
|
||||
# if len(self.__urls) == self.__num_res:
|
||||
# break
|
||||
|
||||
# async def __fetch_url(self: "Google", session: Any, url: str) -> list[str]:
|
||||
# try:
|
||||
# async with session.get(url, headers=HTTP_USERAGENT) as response:
|
||||
# html = await response.text()
|
||||
# soup = BeautifulSoup(html, "html.parser")
|
||||
# text = soup.get_text()
|
||||
# normalized_text = normalizer(text)
|
||||
# sentences: list[str] = sentencizer(normalized_text)
|
||||
# return sentences
|
||||
# except aiohttp.ClientConnectorError:
|
||||
# return [""]
|
||||
# except Exception:
|
||||
# return [""]
|
||||
|
||||
# async def __fetch_urls(self: "Google", urls: list[str]) -> Any:
|
||||
# async with aiohttp.ClientSession() as session:
|
||||
# tasks = [
|
||||
# asyncio.create_task(self.__fetch_url(session, url)) for url in urls
|
||||
# ]
|
||||
# results = await asyncio.gather(*tasks)
|
||||
# return results
|
||||
|
||||
# def __flatten(self: Any, a: list[list[Any]]) -> list[Any]:
|
||||
# return list(itertools.chain(*a))
|
||||
|
||||
# def __get_urls_contents(self: "Google") -> None:
|
||||
# loop = asyncio.new_event_loop()
|
||||
# asyncio.set_event_loop(loop)
|
||||
# contents = loop.run_until_complete(self.__fetch_urls(self.__urls))
|
||||
# loop.close()
|
||||
# self.__content = self.__flatten(contents)
|
||||
|
||||
# def google(self: "Google") -> tuple[list[str], list[str]]:
|
||||
# self.__get_urls()
|
||||
# self.__get_urls_contents()
|
||||
# return (self.__content, self.__urls)
|
||||
|
||||
|
||||
# """
|
||||
# Timing:
|
||||
# import time
|
||||
# start_time = time.time()
|
||||
# google("Who is Elon Musk")
|
||||
# print("--- %s seconds ---" % (time.time() - start_time))
|
||||
|
||||
# # Results:
|
||||
|
||||
# # --- 2.2230100631713867 seconds ---
|
||||
|
||||
# # ________________________________________________________
|
||||
# # Executed in 4.73 secs fish external
|
||||
# # usr time 3.35 secs 85.00 micros 3.35 secs
|
||||
# # sys time 1.86 secs 956.00 micros 1.86 secs
|
||||
# """
|
||||
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
from importlib import reload
|
||||
from pathlib import Path
|
||||
|
|
@ -17,18 +153,22 @@ sys.path.append(str(Path(__file__).parent.parent.parent.parent) + "/utils")
|
|||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import itertools
|
||||
import re
|
||||
|
||||
import aiohttp
|
||||
import config
|
||||
from adremover import AdRemover
|
||||
from bs4 import BeautifulSoup
|
||||
from keywords import get_keywords
|
||||
from normalize import normalizer
|
||||
|
||||
# from relevancy import filter_irrelevant
|
||||
from relevancy import filter_relevant
|
||||
from sentencize import sentencizer
|
||||
from urlextract import URLExtract
|
||||
|
||||
dotenv.load_dotenv()
|
||||
|
||||
|
||||
class Google:
|
||||
def __init__(
|
||||
|
|
@ -37,9 +177,18 @@ class Google:
|
|||
GOOGLE_SEARCH_API_KEY: str,
|
||||
GOOGLE_SEARCH_ENGINE_ID: str,
|
||||
) -> None:
|
||||
self.__GOOGLE_SEARCH_API_KEY: str = GOOGLE_SEARCH_API_KEY
|
||||
self.__GOOGLE_SEARCH_ENGINE_ID: str = GOOGLE_SEARCH_ENGINE_ID
|
||||
self.__num_res: int = 10
|
||||
# if environment keys are not given, assume it is in env
|
||||
if GOOGLE_SEARCH_API_KEY == "":
|
||||
self.__GOOGLE_SEARCH_API_KEY = str(os.environ.get("GOOGLE_SEARCH_API_KEY"))
|
||||
if GOOGLE_SEARCH_ENGINE_ID == "":
|
||||
self.__GOOGLE_SEARCH_ENGINE_ID = str(
|
||||
os.environ.get("GOOGLE_SEARCH_ENGINE_ID")
|
||||
)
|
||||
self.__num_res: int = (
|
||||
5
|
||||
if config.NLP_CONF_MODE == "speed"
|
||||
else (20 if config.NLP_CONF_MODE else 10)
|
||||
)
|
||||
self.__query = query
|
||||
self.__URL_EXTRACTOR: URLExtract = URLExtract()
|
||||
self.__urls: list[str] = self.__URL_EXTRACTOR.find_urls(query)
|
||||
|
|
@ -50,8 +199,15 @@ class Google:
|
|||
str(self.__query),
|
||||
)
|
||||
)
|
||||
self.__content: list[str] = []
|
||||
ADBLOCK_RULES = [
|
||||
"https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt",
|
||||
"https://filters.adtidy.org/extension/chromium/filters/1.txt",
|
||||
]
|
||||
self.__ad_remover = AdRemover(ADBLOCK_RULES)
|
||||
|
||||
def __get_urls(self: "Google") -> None:
|
||||
# Send the request to the Google Search API
|
||||
if self.__GOOGLE_SEARCH_API_KEY == "":
|
||||
exit("ERROR: Google API Key not found")
|
||||
if self.__GOOGLE_SEARCH_ENGINE_ID == "":
|
||||
|
|
@ -74,6 +230,7 @@ class Google:
|
|||
try:
|
||||
async with session.get(url, headers=HTTP_USERAGENT) as response:
|
||||
html = await response.text()
|
||||
html = self.__ad_remover.remove_ads(html)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
text = soup.get_text()
|
||||
normalized_text = normalizer(text)
|
||||
|
|
@ -101,11 +258,26 @@ class Google:
|
|||
contents = loop.run_until_complete(self.__fetch_urls(self.__urls))
|
||||
loop.close()
|
||||
self.__content = self.__flatten(contents)
|
||||
self.__content = [str(x) for x in self.__content]
|
||||
|
||||
def google(self: "Google") -> tuple[list[str], list[str]]:
|
||||
def __filter_irrelevant_processing(self: "Google") -> None:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=500) as executor:
|
||||
futures = [executor.submit(filter_relevant, self.__content, self.__query)]
|
||||
concurrent.futures.wait(futures)
|
||||
content: list[str] = []
|
||||
for future in futures:
|
||||
content.append(future.result())
|
||||
self.__content = content
|
||||
|
||||
def google(
|
||||
self: "Google", filter_irrelevant: bool = True
|
||||
) -> tuple[list[str], list[str]]:
|
||||
self.__get_urls()
|
||||
self.__get_urls_contents()
|
||||
return (self.__content, self.__urls)
|
||||
if filter_irrelevant:
|
||||
self.__filter_irrelevant_processing()
|
||||
results: tuple[list[str], list[str]] = (self.__content, self.__urls)
|
||||
return results
|
||||
|
||||
|
||||
"""
|
||||
|
|
|
|||
Loading…
Reference in New Issue