From 6245f6d0a1d06a4814e930b028d028ed196a7c8e Mon Sep 17 00:00:00 2001 From: Ilia Miheev Date: Sun, 27 Apr 2025 08:38:25 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D1=8B=20=D0=BC=D0=B5=D1=82=D0=BE=D0=B4=D1=8B=20=D0=B4?= =?UTF-8?q?=D0=BB=D1=8F=20=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D1=8B=20=D1=81=20?= =?UTF-8?q?json?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ipars/module.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/ipars/module.py b/ipars/module.py index 976c93c..0472034 100644 --- a/ipars/module.py +++ b/ipars/module.py @@ -4,33 +4,47 @@ from selenium import webdriver from time import sleep from os import remove from bs4 import BeautifulSoup +import json class Pars: '''Библиотека для работы с файлами во время парсинга''' + def __init__(self, url, pathToSaveFile, writeMethod='w'): """Конструктор""" self.url = url self.pathToSaveFile = pathToSaveFile self.writeMethod = writeMethod - def returnBs4Object(self, userEncoding='utf8'): + + def loadJson(self, pathToJsonFile:str): + """Получаем данные из json файла""" + with open(pathToJsonFile) as jsonFile: + src = json.load(jsonFile) + return src + + + def dumpJson(self, data:any, pathToJsonFile:str): + """Записываем данные в json файл""" + with open(pathToJsonFile, 'w') as jsonFile: + json.dump(data, jsonFile, indent=4, ensure_ascii=0) + + + def returnBs4Object(self, myEncoding:str='utf8', parser:str='lxml'): """Возвращаем объект beautifulsoup""" - with open(self.pathToSaveFile, encoding=userEncoding) as file: + with open(self.pathToSaveFile, encoding=myEncoding) as file: src = file.read() - soup = BeautifulSoup(src, 'lxml') + soup = BeautifulSoup(src, parser) return soup + def deleteFile(self): """Удаляем файл после работы с ним""" remove(self.pathToSaveFile) - def get_static_page(self): + def get_static_page(self, headers:dict): '''Получаем статическую страницу''' - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - } try: # Отправляем запрос req = get(self.url, headers=headers) @@ -59,13 +73,13 @@ class Pars: def get_dinamic_page(self, closeWindow:bool=1): - '''Функция для получения динамической страницы''' - + '''Получаем динамическую страницу''' # Устанавливаем опции для Chrome WebDriver options = Options() if closeWindow: options.add_argument('--headless') + # открываем браузер with webdriver.Chrome(options=options) as driver: driver.get(self.url) @@ -85,4 +99,4 @@ class Pars: html_content = driver.page_source # Сохраняем HTML-код в файл with open(self.pathToSaveFile, "w", encoding="utf-8") as file: - file.write(html_content) \ No newline at end of file + file.write(html_content)