alex9ufo 聰明人求知心切: 1月 2023

2023年1月11日星期三

Python爬蟲

源自於 https://pala.tw/python-web-crawler/

import requests

from bs4 import BeautifulSoup

tag = input("請輸入定位元素，class前面加上.，id前面加上# ")

res = requests.get('https://hipala.github.io/class-id-example/')

soup = BeautifulSoup(res.text, "html")

for drink in soup.select('{}'.format(tag)):

print(drink.get_text())

>>> %Run -c $EDITOR_CONTENT

請輸入定位元素，class前面加上.，id前面加上# .半糖

老闆的半糖去冰珍珠奶茶

心儀同事的半糖去冰文山青茶

工讀生的半糖微冰珍珠紅茶

>>>

https://hipala.github.io/class-id-example/

老闆的半糖去冰珍珠奶茶

資深前輩的無糖去冰綠茶

Pala的無糖微冰鐵觀音

心儀同事的半糖去冰文山青茶

工讀生的半糖微冰珍珠紅茶

<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>辦公室的下午茶</title>
</head>
<body>
<p id="老闆" class="半糖去冰珍珠奶茶">老闆的半糖去冰珍珠奶茶</p>
<p id="資深前輩" class="無糖去冰綠茶">資深前輩的無糖去冰綠茶</p>
<p id="Pala" class="無糖微冰鐵觀音">Pala的無糖微冰鐵觀音</p>
<p id="心儀同事" class="半糖去冰文山青茶">心儀同事的半糖去冰文山青茶</p>
<p id="工讀生" class="半糖微冰珍珠紅茶">工讀生的半糖微冰珍珠紅茶</p>
</body>
</html>	ample/')	soup = BeautifulSoup(res.text, "html")	for drink in soup.select('{}'.format(tag)):	print(drink.get_text())

Python 爬取統一發票號碼

源自於 https://steam.oxxostudio.tw/category/python/spider/invoice-auto.html

import requests

url = 'https://invoice.etax.nat.gov.tw/index.html'

web = requests.get(url) # 取得網頁內容

web.encoding='utf-8' # 因為該網頁編碼為 utf-8，加上 .encoding 避免亂碼

from bs4 import BeautifulSoup

soup = BeautifulSoup(web.text, "html.parser") # 轉換成標籤樹

td = soup.select('.container-fluid')[0].select('.etw-tbiggest') # 取出中獎號碼的位置

ns = td[0].getText() # 特別獎

n1 = td[1].getText() # 特獎

# 頭獎，因為存入串列會出現 /n 換行符，使用 [-8:] 取出最後八碼

n2 = [td[2].getText()[-8:], td[3].getText()[-8:], td[4].getText()[-8:]]

print(ns)

print(n1)

print(n2)

>>> %Run invoice_2.py

11174120

59276913

['18079936', '20591738', '64500205']

>>>

2023年1月10日星期二

Pytthon 爬取臺灣銀行牌告匯率 --> Line Notify , Node-red MQTT

Pytthon 爬取臺灣銀行牌告匯率

源自於https://steam.oxxostudio.tw/category/python/spider/exchange.html

import requests

import paho.mqtt.client as mqtt #import the client1

import time

publish=[]

#=================================#

def on_message(client, userdata, message):

print("message received " ,str(message.payload.decode("utf-8")))

print("message topic=",message.topic)

print("message qos=",message.qos)

print("message retain flag=",message.retain)

#=================================#

broker_address="broker.mqtt-dashboard.com"

#broker_address="iot.eclipse.org"

print("creating new instance")

client = mqtt.Client("P1") #create new instance

client.on_message=on_message #attach function to callback

print("connecting to broker --> ",broker_address )

client.connect(broker_address) #connect to broke

#=================================#

url = 'https://rate.bot.com.tw/xrt/flcsv/0/day' # 牌告匯率 CSV 網址

# https://rate.bot.com.tw/xrt/flcsv/0/day

rate = requests.get(url) # 爬取網址內容

rate.encoding = 'utf-8' # 調整回應訊息編碼為 utf-8，避免編碼不同造成亂碼

rt = rate.text # 以文字模式讀取內容

rts = rt.split('\n') # 使用「換行」將內容拆分成串列

#============MQTT===================#

client.loop_start() #start the loop

#============Line Notify===================#

url = 'https://notify-api.line.me/api/notify' # LINE Notify API 網址

#token = '你的 LINE Notify 權杖' # 自己申請的 LINE Notify 權杖

token = 'A4wwPNh2WqB7dlfeQyyIAwtggn1kfZSI5LkkCdia1gB'

headers = {

'Authorization': 'Bearer ' + token # POST 使用的 headers

}

for i in rts: # 讀取串列的每個項目

try: # 使用 try 避開最後一行的空白行

a = i.split(',') # 每個項目用逗號拆分成子串列

print(a[0] + ': ' + a[12]) # 取出第一個 ( 0 ) 和第十三個項目 ( 12 )

#--------------------------------------------------------------------

client.publish("alex9ufo/ratetbot",a[0] + ': ' + a[12])

#--------------------------------------------------------------------

data = {# 發送的訊息

'message':'Python送出目前台灣銀行牌告匯率 -->幣別現金 '+ a[0] + ': ' + a[12]

}

data = requests.post(url, headers=headers, data=data) # 發送 LINE NOtify

time.sleep(5) # wait

#--------------------------------------------------------------------

except:

break

#============MQTT===================#

client.loop_stop() #stop the loop

#============Line Notify===================#

>>> %Run Rate_2.py

creating new instance

connecting to broker --> broker.mqtt-dashboard.com

幣別: 現金

USD: 30.70000

HKD: 3.94300

GBP: 37.90000

AUD: 21.36000

CAD: 23.06000

SGD: 23.17000

CHF: 33.35000

JPY: 0.23340

ZAR: 0.00000

SEK: 3.05000

NZD: 19.76000

THB: 0.96850

PHP: 0.61960

IDR: 0.00233

EUR: 33.18000

KRW: 0.02652

VND: 0.00148

MYR: 7.43400

CNY: 4.54300

>>>

[{"id":"d3c84f39d1691d3c","type":"debug","z":"323770202b3d133d","name":"debug 58","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":480,"y":500,"wires":[]},{"id":"cc243efbb1f863bb","type":"mqtt in","z":"323770202b3d133d","name":"","topic":"alex9ufo/ratetbot","qos":"2","datatype":"auto-detect","broker":"841df58d.ee5e98","nl":false,"rap":true,"rh":0,"inputs":0,"x":240,"y":500,"wires":[["d3c84f39d1691d3c","d498ad7ef323948c"]]},{"id":"d498ad7ef323948c","type":"ui_text","z":"323770202b3d133d","group":"a73c0688.26bac8","order":1,"width":0,"height":0,"name":"","label":"牌告匯率","format":"{{msg.payload}}","layout":"row-left","className":"","x":480,"y":540,"wires":[]},{"id":"841df58d.ee5e98","type":"mqtt-broker","name":"","broker":"broker.mqtt-dashboard.com","port":"1883","clientid":"","usetls":false,"compatmode":false,"keepalive":"15","cleansession":true,"birthTopic":"","birthQos":"0","birthPayload":"","closeTopic":"","closePayload":"","willTopic":"","willQos":"0","willPayload":""},{"id":"a73c0688.26bac8","type":"ui_group","name":"Bits","tab":"207e613d.22c52e","order":4,"disp":false,"width":10,"collapse":false},{"id":"207e613d.22c52e","type":"ui_tab","name":"Configuration","icon":"dashboard","order":5,"disabled":false,"hidden":false}]

Python 爬取空氣品質指標 ( AQI )

Python 爬取空氣品質指標 ( AQI )

import requests

# 2022/12 時氣象局有修改了 API 內容，將部份大小寫混合全改成小寫，因此程式碼也跟著修正

url = 'https://data.epa.gov.tw/api/v2/aqx_p_432?api_key=e8dd42e6-9b8b-43f8-991e-b3dee723a52d&limit=1000&sort=ImportDate%20desc&format=JSON'

data = requests.get(url) # 使用 get 方法透過空氣品質指標 API 取得內容

data_json = data.json() # 將取得的檔案轉換為 JSON 格式

for i in data_json['records']: # 依序取出 records 內容的每個項目

print(i['county'] + ' ' + i['sitename'], end='，') # 印出城市與地點名稱

print('AQI:' + i['aqi'], end='，') # 印出 AQI 數值

print('空氣品質' + i['status']) # 印出空氣品質狀態

>>> %Run AQI_1.py

基隆市基隆，AQI:31，空氣品質良好

新北市汐止，AQI:26，空氣品質良好

新北市萬里，AQI:38，空氣品質良好

新北市新店，AQI:28，空氣品質良好

新北市土城，AQI:32，空氣品質良好

新北市板橋，AQI:33，空氣品質良好

新北市新莊，AQI:28，空氣品質良好

新北市菜寮，AQI:28，空氣品質良好

新北市林口，AQI:26，空氣品質良好

新北市淡水，AQI:17，空氣品質良好

臺北市士林，AQI:21，空氣品質良好

臺北市中山，AQI:33，空氣品質良好

臺北市萬華，AQI:32，空氣品質良好

臺北市古亭，AQI:36，空氣品質良好

臺北市松山，AQI:30，空氣品質良好

臺北市大同，AQI:40，空氣品質良好

桃園市桃園，AQI:27，空氣品質良好

桃園市大園，AQI:37，空氣品質良好

桃園市觀音，AQI:42，空氣品質良好

桃園市平鎮，AQI:33，空氣品質良好

桃園市龍潭，AQI:39，空氣品質良好

新竹縣湖口，AQI:34，空氣品質良好

新竹縣竹東，AQI:38，空氣品質良好

新竹市新竹，AQI:28，空氣品質良好

苗栗縣頭份，AQI:34，空氣品質良好

苗栗縣苗栗，AQI:46，空氣品質良好

苗栗縣三義，AQI:33，空氣品質良好

臺中市豐原，AQI:59，空氣品質普通

臺中市沙鹿，AQI:48，空氣品質良好

臺中市大里，AQI:63，空氣品質普通

臺中市忠明，AQI:57，空氣品質普通

臺中市西屯，AQI:55，空氣品質普通

彰化縣彰化，AQI:55，空氣品質普通

彰化縣線西，AQI:58，空氣品質普通

彰化縣二林，AQI:64，空氣品質普通

南投縣南投，AQI:72，空氣品質普通

雲林縣斗六，AQI:94，空氣品質普通

雲林縣崙背，AQI:84，空氣品質普通

嘉義縣新港，AQI:96，空氣品質普通

嘉義縣朴子，AQI:87，空氣品質普通

雲林縣臺西，AQI:61，空氣品質普通

嘉義市嘉義，AQI:105，空氣品質對敏感族群不健康

臺南市新營，AQI:89，空氣品質普通

臺南市善化，AQI:87，空氣品質普通

臺南市安南，AQI:78，空氣品質普通

臺南市臺南，AQI:94，空氣品質普通

高雄市美濃，AQI:88，空氣品質普通

高雄市橋頭，AQI:91，空氣品質普通

高雄市仁武，AQI:108，空氣品質對敏感族群不健康

高雄市鳳山，AQI:113，空氣品質對敏感族群不健康

高雄市大寮，AQI:117，空氣品質對敏感族群不健康

高雄市林園，AQI:99，空氣品質普通

高雄市楠梓，AQI:99，空氣品質普通

高雄市左營，AQI:83，空氣品質普通

高雄市前金，AQI:86，空氣品質普通

高雄市前鎮，AQI:89，空氣品質普通

高雄市小港，AQI:99，空氣品質普通

屏東縣屏東，AQI:111，空氣品質對敏感族群不健康

屏東縣潮州，AQI:116，空氣品質對敏感族群不健康

屏東縣恆春，AQI:35，空氣品質良好

臺東縣臺東，AQI:32，空氣品質良好

花蓮縣花蓮，AQI:28，空氣品質良好

臺北市陽明，AQI:36，空氣品質良好

宜蘭縣宜蘭，AQI:28，空氣品質良好

宜蘭縣冬山，AQI:24，空氣品質良好

新北市三重，AQI:51，空氣品質普通

桃園市中壢，AQI:48，空氣品質良好

南投縣竹山，AQI:91，空氣品質普通

新北市永和，AQI:36，空氣品質良好

高雄市復興，AQI:85，空氣品質普通

南投縣埔里，AQI:75，空氣品質普通

連江縣馬祖，AQI:106，空氣品質對敏感族群不健康

金門縣金門，AQI:69，空氣品質普通

澎湖縣馬公，AQI:70，空氣品質普通

臺東縣關山，AQI:29，空氣品質良好

雲林縣麥寮，AQI:61，空氣品質普通

新北市富貴角，AQI:45，空氣品質良好

彰化縣大城，AQI:52，空氣品質普通

彰化縣彰化(員林)，AQI:57，空氣品質普通

高雄市高雄(湖內)，AQI:89，空氣品質普通

臺南市臺南(麻豆)，AQI:78，空氣品質普通

屏東縣屏東(琉球)，AQI:85，空氣品質普通

宜蘭縣宜蘭(壯圍)，AQI:37，空氣品質良好

新北市新北(樹林)，AQI:42，空氣品質良好

臺南市臺南(學甲)，AQI:76，空氣品質普通

屏東縣屏東(枋山)，AQI:63，空氣品質普通

>>>

"records": [
        {
            "sitename": "基隆",
            "county": "基隆市",
            "aqi": "30",
            "pollutant": "",
            "status": "良好",
            "so2": "1.5",
            "co": "0.2",
            "o3": "31.9",
            "o3_8hr": "32.9",
            "pm10": "9",
            "pm2.5": "",
            "no2": "5.6",
            "nox": "5.8",
            "no": "0.2",
            "wind_speed": "0.3",
            "wind_direc": "35",
            "publishtime": "2023\/01\/10 21:00:00",
            "co_8hr": "0.1",
            "pm2.5_avg": "",
            "pm10_avg": "13",
            "so2_avg": "1",
            "longitude": "121.760056",
            "latitude": "25.129167",
            "siteid": "1"
        },

Python 透過 LINE Notify 與 MQTT--> Node-red LINE Notify傳送雷達回波圖

Python 透過 LINE Notify 與 MQTT--> Node-red LINE Notify傳送雷達回波圖

源自於https://steam.oxxostudio.tw/category/python/spider/radar.html

import requests

import paho.mqtt.client as mqtt #import the client1

import time

publish=''

def on_message(client, userdata, message):

print("message received " ,str(message.payload.decode("utf-8")))

print("message topic=",message.topic)

print("message qos=",message.qos)

print("message retain flag=",message.retain)

########################################

broker_address="broker.mqtt-dashboard.com"

#broker_address="iot.eclipse.org"

print("creating new instance")

client = mqtt.Client("P1") #create new instance

client.on_message=on_message #attach function to callback

print("connecting to broker --> ",broker_address )

client.connect(broker_address) #connect to broke

########################################

radar_url = 'https://opendata.cwb.gov.tw/fileapi/v1/opendataapi/O-A0058-003?Authorization=rdec-key-123-45678-011121314&format=JSON'

radar = requests.get(radar_url) # 爬取資料

radar_json = radar.json() # 使用 JSON 格式

radar_img = radar_json['cwbopendata']['dataset']['resource']['uri'] # 取得圖片網址

print(radar_img)

'''

msg.headers = {'content-type':'application/x-www-form-urlencoded','Authorization':'Bearer A4wwPNh2WqB7dlfeQyyIAwtggn1kfZSI5LkkCdia1gB'};

msg.payload = {"message":msg.payload};

return msg;

'''

url = 'https://notify-api.line.me/api/notify' # LINE Notify API 網址

#token = '你的 LINE Notify 權杖' # 自己申請的 LINE Notify 權杖

token = 'A4wwPNh2WqB7dlfeQyyIAwtggn1kfZSI5LkkCdia1gB'

headers = {

'Authorization': 'Bearer ' + token # POST 使用的 headers

}

data = {

'message':'Python送出從雷達回波看看會不會下雨～', # 發送的訊息

'imageThumbnail':radar_img, # 預覽圖網址

'imageFullsize':radar_img # 完整圖片網址

}

data = requests.post(url, headers=headers, data=data) # 發送 LINE NOtify

publish=radar_img

client.loop_start() #start the loop

client.publish("alex9ufo/weather/O-A0058-003",publish)

time.sleep(10) # wait

client.loop_stop() #stop the loop

>>> %Run weather_6.py

creating new instance

connecting to broker --> broker.mqtt-dashboard.com

https://cwbopendata.s3.ap-northeast-1.amazonaws.com/MSC/O-A0058-003.png

>>>

[{"id":"9829271aabbe6547","type":"http request","z":"1257c22baca94adc","name":"","method":"GET","ret":"bin","paytoqs":"ignore","url":"https://cwbopendata.s3.ap-northeast-1.amazonaws.com/MSC/O-A0058-003.png","tls":"","persist":false,"proxy":"","insecureHTTPParser":false,"authType":"","senderr":false,"headers":[],"x":260,"y":200,"wires":[["4bdb85e00d197670"]]},{"id":"e7765b8d9c73972d","type":"inject","z":"1257c22baca94adc","name":"","repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"str","x":110,"y":240,"wires":[["9829271aabbe6547"]]},{"id":"4bdb85e00d197670","type":"base64","z":"1257c22baca94adc","name":"","action":"","property":"payload","x":420,"y":200,"wires":[["5efea32a70b22095","4416caaaf0b9114e"]]},{"id":"5efea32a70b22095","type":"image","z":"1257c22baca94adc","name":"","width":160,"data":"payload","dataType":"msg","thumbnail":true,"active":true,"outputs":0,"x":620,"y":200,"wires":[]},{"id":"047993ea0d909a53","type":"mqtt in","z":"1257c22baca94adc","name":"","topic":"alex9ufo/weather/O-A0058-003","qos":"2","datatype":"auto-detect","broker":"841df58d.ee5e98","nl":false,"rap":true,"rh":0,"inputs":0,"x":170,"y":120,"wires":[["8936ddaa7a14dd29","9829271aabbe6547"]]},{"id":"8936ddaa7a14dd29","type":"debug","z":"1257c22baca94adc","name":"debug 57","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":460,"y":120,"wires":[]},{"id":"4416caaaf0b9114e","type":"delay","z":"1257c22baca94adc","name":"","pauseType":"delay","timeout":"5","timeoutUnits":"seconds","rate":"1","nbRateUnits":"1","rateUnits":"second","randomFirst":"1","randomLast":"5","randomUnits":"seconds","drop":false,"allowrate":false,"outputs":1,"x":260,"y":300,"wires":[["10685170bcc4d2b2"]]},{"id":"10685170bcc4d2b2","type":"line-notify","z":"1257c22baca94adc","name":"","message":"send image 從Node-Red","contentType":"imageUrl","imageThumbnail":"","imageUrl":"https://cwbopendata.s3.ap-northeast-1.amazonaws.com/MSC/O-A0058-003.png","sticker":"default","stickerPackageId":"1","stickerId":"1","silent":false,"creds":"4d14683c16fc0999","x":400,"y":300,"wires":[["af93dbe2dfce13e9"]]},{"id":"af93dbe2dfce13e9","type":"image","z":"1257c22baca94adc","name":"","width":160,"data":"payload","dataType":"msg","thumbnail":true,"active":true,"outputs":0,"x":580,"y":360,"wires":[]},{"id":"841df58d.ee5e98","type":"mqtt-broker","name":"","broker":"broker.mqtt-dashboard.com","port":"1883","clientid":"","usetls":false,"compatmode":false,"keepalive":"15","cleansession":true,"birthTopic":"","birthQos":"0","birthPayload":"","closeTopic":"","closePayload":"","willTopic":"","willQos":"0","willPayload":""},{"id":"4d14683c16fc0999","type":"linetoken","name":"alex line"}]

2023年1月9日星期一

Python 天氣 --> Node-Red MQTT --> Line Notifly

Python 天氣 --> Node-Red MQTT --> Line Notifly

import requests

import paho.mqtt.client as mqtt #import the client1

import time

publish=''

j=1

def on_message(client, userdata, message):

print("message received " ,str(message.payload.decode("utf-8")))

print("message topic=",message.topic)

print("message qos=",message.qos)

print("message retain flag=",message.retain)

########################################

broker_address="broker.mqtt-dashboard.com"

#broker_address="iot.eclipse.org"

print("creating new instance")

client = mqtt.Client("P1") #create new instance

client.on_message=on_message #attach function to callback

print("connecting to broker")

client.connect(broker_address) #connect to broke

# url = '一般天氣預報 - 今明 36 小時天氣預報 JSON 連結'

url = "https://opendata.cwb.gov.tw/fileapi/v1/opendataapi/F-C0032-001?Authorization=rdec-key-123-45678-011121314&format=JSON"

data = requests.get(url) # 取得 JSON 檔案的內容為文字

data_json = data.json() # 轉換成 JSON 格式

location = data_json['cwbopendata']['dataset']['location']

client.loop_start() #start the loop

for i in location:

city = i['locationName'] # 縣市名稱

wx8 = i['weatherElement'][0]['time'][0]['parameter']['parameterName'] # 天氣現象

maxt8 = i['weatherElement'][1]['time'][0]['parameter']['parameterName'] # 最高溫

mint8 = i['weatherElement'][2]['time'][0]['parameter']['parameterName'] # 最低溫

ci8 = i['weatherElement'][3]['time'][0]['parameter']['parameterName'] # 舒適度

pop8 = i['weatherElement'][4]['time'][0]['parameter']['parameterName'] # 降雨機率

print(f'{j} {city}未來 8 小時{wx8}，最高溫 {maxt8} 度，最低溫 {mint8} 度，降雨機率 {pop8} %')

publish=(f'{j} {city}未來 8 小時{wx8}，最高溫 {maxt8} 度，最低溫 {mint8} 度，降雨機率 {pop8} %')

j=j+1

print("Subscribing to topic","alex9ufo/weather")

client.subscribe("alex9ufo/weather")

client.publish("alex9ufo/weather",publish)

time.sleep(10) # wait

client.loop_stop() #stop the loop

>>> %Run -c $EDITOR_CONTENT

creating new instance

connecting to broker

1 臺北市未來 8 小時陰時多雲短暫雨，最高溫 20 度，最低溫 18 度，降雨機率 70 %