Online programming tutor in Houston

Data scraping techniques

import sqlite3
import requests
from bs4 import BeautifulSoup as bs
def scrap_ticker_symbols():
url = 'https://money.cnn.com/data/hotstocks/'
# Download HTML content.
response = requests.get(url)
# Check status code. for example if the URL page isn't found the server
# would response with a 404 status code
if response.status_code % 1000 // 100 != 2:
# The http response is different than 2xx
raise Exception("Error downloading html content")
# Parse the HTML content using SOAP module
soap = bs(response.content, 'html.parser')
table_active_stocks = soap.find('table', attrs={'class': 'wsod_dataTableBigAlt'})
# Find all HTML elements in page.
td_elements = table_active_stocks.find_all('td')
# Find all  elements within the above elements above with class wsod_symbol
a_elements = [
element.find('a', attrs={'class':'wsod_symbol'})
for element in td_elements
]
# Return the text inside the  elements.
return [a_element.text for a_element in a_elements if a_element]
def scrap_stock_details(ticker_symbol: str):
print(f'Scrapping ticker symbol {ticker_symbol} details')
url = f'https://finance.yahoo.com/quote/{ticker_symbol}?p={ticker_symbol}'
# Download HTML content.
response = requests.get(url)
# Check status code. for example if the URL page isn't found the server
# would response with a 404 status code
if response.status_code % 1000 // 100 != 2:
# The http response is different than 2xx
raise Exception("Error downloading html content")
# Parse the HTML content using SOAP module
soap = bs(response.content, 'html.parser')
try:
# The open price is inside a element with class OPEN-value'
open_price = float(soap.find('td', attrs={'data-test':'OPEN-value'}).span.text)
# The average volumn is inside a element with class AVERAGE_VOLUME_3MONTH-value'
avg_volume_str = soap.find('td', attrs={'data-test':'AVERAGE_VOLUME_3MONTH-value'}).span.text
# Replace all the commas inside the average volumn text so we can parse it as int.
avg_volume = int(avg_volume_str.replace(',', ''))
# The PE ratio is inside a element with class PE_RATIO-value'
pe_ratio_str = soap.find('td', attrs={'data-test':'PE_RATIO-value'}).span.text
pe_ratio = float(pe_ratio_str) if pe_ratio_str != 'N/A' else None
return ticker_symbol, open_price, avg_volume, pe_ratio
except:
return False
def save_data_db(db_name: str, data: list):
# Establish connection to sqllite database db_name
connection = sqlite3.connect(db_name)
# Create a cursor.
cursor = connection.cursor()
# Data definition language to create table.
ddl_sql = """
CREATE TABLE IF NOT EXISTS Stocks (
Ticker text,
OpenPrice number,
AvgVolume int,
PERatio number
)
"""
# Execute SQL.
cursor.execute(ddl_sql)
# Save changes to database
connection.commit()
for row in data:
if not row:
# Empty row skip.
continue
# Change the value of our metrics to null in case it is missing.
ticker_symbol, open_price, avg_volume, pe_ratio = map(
lambda value: value if value else 'null', row
)
# build query to insert row.
query = f"INSERT INTO Stocks (Ticker, OpenPrice, AvgVolume, PERatio) VALUES ('{ticker_symbol}', {open_price}, {avg_volume}, {pe_ratio})"
# Execute query to insert row.
cursor.execute(query)
# Save changes to database.
connection.commit()
def save_data_txt(filename: str, data: list):
is_header = True
with open(filename, 'w') as f:
for stock_details in data:
if is_header:
# Save header name.
line = 'Ticker,OpenPrice,AvgVolume,PERatio\n'
is_header = False
f.write(line)
# Write a symbol per line.
line = ','.join(str(value) if value else '' for value in stock_details) + '\n'
f.write(line)
def main():
print('Scrapping ticker symobols')
ticker_symbols = scrap_ticker_symbols()
data = list()
for symbol in ticker_symbols:
data.append(scrap_stock_details(symbol))
db_name = 'StocksDatabse.db'
print(f'\nWriting data to database {db_name}')
save_data_db(db_name, data)
csv_filename = 'stocks.txt'
print(f'\nSaving ticker symobols to file {csv_filename}')
save_data_txt(csv_filename, data)
print('Scrapping done')
if __name__ == "__main__":
main()

Creating Tables using Python

#• Show some info
print("You must enter the name of the asset, the year when it was pourchased, its cost, its salvage value and the useful life.")
print("For the year of purchase, its cost and salve value, you must enter a numeric value.")
print("You can enter the costs with the currency symbol or without it (i.e: $1000 or 1000).")
# Ask for item name
itemName = input("Enter name of item: ")
"""
In the next loops, the variable 'invalidOption'
is used to know when to stop the while loops
"""
# First while loop to ensure that user enters a correct year
invalidOption = 1
while(invalidOption == 1):
yearPurchased = input("Enter year purchased: ")
if yearPurchased.isnumeric() and int(yearPurchased) > 0:
yearPurchased = int(yearPurchased)
invalidOption = 0
else:
print("You entered an invalid value. Please enter a valid year.")
# Second while loop to ensure that user enters a correct value
invalidOption = 1
while(invalidOption == 1):
cost = input("Enter cost: ")
if '$' in cost:
cost = cost[1:]
if cost.isnumeric() and float(cost) > 0:
cost = float(cost)
invalidOption = 0
else:
print("You entered an invalid value. Please enter a numeric value higher than zero.")
# Third loop used to ensure that user enters a valid salvage value
invalidOption = 1
while(invalidOption == 1):
salvageValue = input("Enter salvage value: ")
if '$' in salvageValue:
salvageValue = salvageValue[1:]
if salvageValue.isnumeric() and float(salvageValue) > 0:
salvageValue = float(salvageValue)
invalidOption = 0
else:
print("You entered an invalid value. Please enter a numeric value higher than zero.")
# Last loop used to ensure that user enters a valid asset life
invalidOption = 1
while(invalidOption == 1):
life = input("Enter estimated life (in years): ")
if life.isnumeric() and int(life) > 0:
life = int(life)
invalidOption = 0
else:
print("You entered an invalid value. Please enter a estimated life for the asset.")
# Calculate depreciation value
depreciation = (cost - salvageValue)/life
# Print description
print("Description: " + itemName)
print("Year of purchase: " + str(yearPurchased))
print("Cost: $" + str(cost))
print("Estimated life: " + str(life))
print("Salvage Value: $" + str(salvageValue))
print("Method of depreciation: Straight-Line")
print("")
print("{:>40}".format("DEPRECIATION TABLE"))
print("{:>40}".format("------------------"))
# Print header
print("{:>20} {:>15} {:>15} {:>15}".format("Beginning", "Amount of", "Total", "Ending"))
print("{:>0} {:>13} {:>18} {:>18} {:>11}".format("Year", "Value", "Depreciation", "Depreciation", "Value"))
print("{:>0} {:>15} {:>16} {:>18} {:>12}".format("----", "---------", "------------", "------------", "--------"))
value = cost
totalDepreciation = 0
k = 0
for i in range(life):
currentYear = yearPurchased + k
endingValue = value - depreciation
totalDepreciation += depreciation
print("{:>0} {:>15,.2f} {:>13,.2f} {:>18,.2f} {:>16,.2f}".format(currentYear, value, depreciation, totalDepreciation, endingValue))
value = value - depreciation
k+=1

Python Homework Help Expert

Stacy L

Profession

Skills

Python Functional Programming Specialist

Classes and Objects Homework Solver

Adept Persistence and Databases Professional

Seasoned Expert in Network Programming with Python

Python Threads and Concurrency Wizard

Data scraping techniques

Creating Tables using Python