Python Tests

This commit is contained in:
YuCheng Hu 2023-09-20 00:22:26 -04:00
parent 3986a5c636
commit f42bdb233f
18 changed files with 245 additions and 24 deletions

16
.idea/checkstyle-idea.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CheckStyle-IDEA" serialisationVersion="2">
<checkstyleVersion>10.12.2</checkstyleVersion>
<scanScope>JavaOnly</scanScope>
<copyLibs>true</copyLibs>
<option name="thirdPartyClasspath" />
<option name="activeLocationIds" />
<option name="locations">
<list>
<ConfigurationLocation id="bundled-sun-checks" type="BUNDLED" scope="All" description="Sun Checks">(bundled)</ConfigurationLocation>
<ConfigurationLocation id="bundled-google-checks" type="BUNDLED" scope="All" description="Google Checks">(bundled)</ConfigurationLocation>
</list>
</option>
</component>
</project>

View File

@ -1,7 +0,0 @@
<component name="ProjectCodeStyleConfiguration">
<code_scheme name="Project" version="173">
<ScalaCodeStyleSettings>
<option name="MULTILINE_STRING_CLOSING_QUOTES_ON_NEW_LINE" value="true" />
</ScalaCodeStyleSettings>
</code_scheme>
</component>

View File

@ -1,5 +0,0 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
</state>
</component>

12
.idea/dataSources.xml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
<data-source source="LOCAL" name="discourse@nas1120" uuid="30e9cdf1-c2c6-4512-af34-441e5e96dbdf">
<driver-ref>postgresql</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>org.postgresql.Driver</jdbc-driver>
<jdbc-url>jdbc:postgresql://nas1120:5433/discourse</jdbc-url>
<working-dir>$ProjectFileDir$</working-dir>
</data-source>
</component>
</project>

6
.idea/jpa-buddy.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JpaBuddyIdeaProjectConfig">
<option name="renamerInitialized" value="true" />
</component>
</project>

View File

@ -1,6 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (python-tutorials)" project-jdk-type="Python SDK">
<component name="ProjectRootManager" version="2" languageLevel="JDK_20" project-jdk-name="Python 3.11 (python-tutorials)" project-jdk-type="Python SDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
<component name="ProjectType">
<option name="id" value="jpab" />
</component>
<component name="ThriftCompiler">
<compilers />
</component>
</project>

View File

@ -2,7 +2,7 @@
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/python-tutorials.iml" filepath="$PROJECT_DIR$/python-tutorials.iml" />
<module fileurl="file://$PROJECT_DIR$/.idea/python-tutorials.iml" filepath="$PROJECT_DIR$/.idea/python-tutorials.iml" />
</modules>
</component>
</project>

7
.idea/other.xml Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="PySciProjectComponent">
<option name="PY_SCI_VIEW" value="true" />
<option name="PY_SCI_VIEW_SUGGESTED" value="true" />
</component>
</project>

View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="renderExternalDocumentation" value="true" />
</component>
</module>

6
.idea/sqldialects.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$USER_HOME$/Downloads/dump.sql.gz" dialect="PostgreSQL" />
</component>
</project>

View File

@ -2,7 +2,8 @@
欢迎来到 Python 教程和工具库
GitHub 上有关 CWIKIUS 的项目:[https://github.com/cwiki-us-docs/cwikius-docs](https://github.com/cwiki-us-docs/cwikius-docs)
GitHub 上有关 CWIKIUS
的项目:[https://github.com/cwiki-us-docs/cwikius-docs](https://github.com/cwiki-us-docs/cwikius-docs)
[CONTACT](CONTACT.md ':include')
@ -12,10 +13,10 @@ GitHub 上有关 CWIKIUS 的项目:[https://github.com/cwiki-us-docs/cwikius-d
在这里对原作者表示感谢。
| 网站名称 | URL | NOTE |
|---|---|---|
| WWW.CWIKIUS.CN | [http://www.cwikius.cn/](http://www.cwikius.cn/) | CWIKIUS.CN 一个有独立思考和温度的清新站 |
| [Python - 100天从新手到大师](https://github.com/jackfrued/Python-100-Days) | [https://github.com/jackfrued/Python-100-Days](https://github.com/jackfrued/Python-100-Days) | 我们参考了原作者的一些代码和测试用例。 |
| [Python语言基础50课](https://github.com/jackfrued/Python-Core-50-Courses) | [https://github.com/jackfrued/Python-Core-50-Courses](https://github.com/jackfrued/Python-Core-50-Courses) | 参考了原作者的部分内容和编排目录 |
| [Python 脚本收集](https://github.com/fnplus/Python-scripts-collection) | [https://github.com/fnplus/Python-scripts-collection](https://github.com/fnplus/Python-scripts-collection) | 针对 Python 使用的一些实际脚本的收集 |
| 网站名称 | URL | NOTE |
|----------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------|---------------------------|
| WWW.CWIKIUS.CN | [http://www.cwikius.cn/](http://www.cwikius.cn/) | CWIKIUS.CN 一个有独立思考和温度的清新站 |
| [Python - 100天从新手到大师](https://github.com/jackfrued/Python-100-Days) | [https://github.com/jackfrued/Python-100-Days](https://github.com/jackfrued/Python-100-Days) | 我们参考了原作者的一些代码和测试用例。 |
| [Python语言基础50课](https://github.com/jackfrued/Python-Core-50-Courses) | [https://github.com/jackfrued/Python-Core-50-Courses](https://github.com/jackfrued/Python-Core-50-Courses) | 参考了原作者的部分内容和编排目录 |
| [Python 脚本收集](https://github.com/fnplus/Python-scripts-collection) | [https://github.com/fnplus/Python-scripts-collection](https://github.com/fnplus/Python-scripts-collection) | 针对 Python 使用的一些实际脚本的收集 |

View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<module type="WEB_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$" />

View File

@ -6,5 +6,8 @@ ruamel.yaml
qrcode~=6.1
Pillow
pyautogui
requests
numpy
stomp.py
selenium
requests
xlrd

77
tests/DoCcrawler.py Normal file
View File

@ -0,0 +1,77 @@
import time
import json
import stomp
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
# 待查询信息
AMQHOST = "mq.ossez.com"
AMQPORT = 61616
AMQUSER = "artemis"
AMQPASS = "artemis"
TOPICNAME = "policyQueue"
begin_year = input("请输入最早年份:")
current_timestamp = time.time()
time_tuple = time.localtime(current_timestamp)
end_year = time_tuple.tm_year
wait_time = 0.5 # 等待时间
action_pixel = 100 # 鼠标滚动像素
# get网站
chrome = Chrome(service=Service(r"C:\Users\yhu\Downloads\chromedriver-win64\chromedriver-win64\chromedriver.exe"))
chrome.get('https://www.isharkfly.com/')
wait = WebDriverWait(chrome, 10)
rowContent = chrome.find_elements(By.XPATH, '/html/body/div[3]/div/div/div/div[4]/div/table/tbody/tr')
main_window = chrome.current_window_handle
def send_mq(ctxInfo, ctx):
for summary in ctxInfo.splitlines():
if summary.startswith('索引号'):
index_string = summary.split('')[1]
elif summary.startswith('发文日期'):
policyDateTime = summary.split('')[1]
elif summary.startswith('名称'):
policy_name = summary.split('')[1]
data = {}
data['policy_index_number'] = index_string
data['policy_title'] = policy_name
data['policy_content'] = ctx
hosts = [(AMQHOST, AMQPORT)]
conn = stomp.Connection(host_and_ports=hosts, auto_content_length=False)
conn.connect(username=AMQUSER, passcode=AMQPASS, wait=True)
conn.send(body=json.dumps(data), destination=TOPICNAME)
time.sleep(1)
conn.disconnect()
for tr in rowContent:
tdList = tr.find_elements("xpath", 'td')
indexNumber = tdList[0].text
docName = tdList[1].text
tdList[1].find_element(By.TAG_NAME, 'a').click()
docDate = tdList[2].text
wait.until(EC.number_of_windows_to_be(2))
# Loop through until we find a new window handle
for window_handle in chrome.window_handles:
if window_handle != main_window:
chrome.switch_to.window(window_handle)
break
ctxInfo = chrome.find_element(By.XPATH, '/html/body/div[3]/div/div/div/div[2]/div/div[1]/ul').text
ctx = chrome.find_element(By.XPATH, '/html/body/div[3]/div/div/div/div[2]/div/div[2]').text
send_mq(ctxInfo, ctx)
chrome.close()
chrome.switch_to.window(main_window)
print(docName)
# break
chrome.quit()

View File

@ -0,0 +1,69 @@
import time
import requests
import stomp
import json
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
# 待查询信息
AMQHOST = "mq.ossez.com"
AMQPORT = 61616
AMQUSER = "artemis"
AMQPASS = "artemis"
TOPICNAME = "policyQueue"
def send_mq(data):
hosts = [(AMQHOST, AMQPORT)]
conn = stomp.Connection(host_and_ports=hosts, auto_content_length=False)
conn.connect(username=AMQUSER, passcode=AMQPASS, wait=True)
conn.send(body=json.dumps(data), destination=TOPICNAME)
conn.disconnect()
# https://policyapi.10nservice.com/api/v1/WebPolicy/GetSearchPageList?pageSize=50&pageIndex=1&postType=99&release=&years=&area=430100&platformId=3479085520414310401
def do_data_crawl(page_index):
URL = "https://policyapi.10nservice.com/api/v1/WebPolicy/GetSearchPageList"
PARAMS = {'pageSize': 5000, 'pageIndex': page_index, 'postType': 99, 'postType': '', 'years': '', 'area': 430100,
'platformId': 3479085520414310401}
# sending get request and saving the response as response object
r = requests.get(url=URL, params=PARAMS)
# extracting data in json format
responseData = r.json()
# Loop List
for policyList in json.loads(responseData['Data']):
pid = policyList['PID']
policyTitle = policyList['PolicyTitle']
detail_url = "https://policyapi.10nservice.com/api/v1/WebPolicy/GetAdoptDetails?pid=" + pid + "&platformId=3479085520414310401"
request_detail_data = requests.get(url=detail_url).json()
province_id = json.loads(request_detail_data['Data'])['ProvinceID']
data = {}
data['policy_index_number'] = pid
data['policy_title'] = policyTitle
data['policy_content'] = json.loads(request_detail_data['Data'])['PolicyText']
data['policy_tag'] = json.loads(request_detail_data['Data'])['PolicyKey']
data['release_time'] = json.loads(request_detail_data['Data'])['ReleaseTime']
data['start_time'] = json.loads(request_detail_data['Data'])['StarTime']
data['end_time'] = json.loads(request_detail_data['Data'])['EndTime']
data['source_name'] = json.loads(request_detail_data['Data'])['Source']
data['source_url'] = json.loads(request_detail_data['Data'])['PageUrl']
send_mq(data)
print(policyTitle)
# break
for i in range(3, 6):
do_data_crawl(i)
# break

View File

@ -9,7 +9,7 @@ import qrcode.image.svg
image_path = "resources/token_qr.png"
qr_string = "https://www.ossez.com/c/open-source/python/14"
print(qr_string)
print(rebase = hex(base62.decode(base, base62.CHARSET_INVERTED))[2:].zfill(40))
img = qrcode.make(qr_string)
img.save(image_path)

16
tests/StompArtemis.py Normal file
View File

@ -0,0 +1,16 @@
import time
import stomp
AMQHOST = "nas1120"
AMQPORT = 61616
AMQUSER = "artemis"
AMQPASS = "artemis"
TOPICNAME = "remotingQueue"
hosts = [(AMQHOST, AMQPORT)]
conn = stomp.Connection(host_and_ports=hosts, auto_content_length=False)
conn.connect(username=AMQUSER, passcode=AMQPASS, wait=True)
conn.send(body='Love Python', destination=TOPICNAME)
time.sleep(320)
conn.disconnect()

Binary file not shown.

After

Width:  |  Height:  |  Size: 825 B