import json import urllib.parse import time import random from bluegems import bluegems from RequestManager import RequestManager import HelperFunctions import psycopg2 import queue from FinalListing import FinalListing import topfloatdb from LoggingFormatter import logger import SeleniumExecutor import ast class NewMarketBot(object): def __init__( self, high_low: str, weapon_name: str, webshare_ips: list, all_sessions: list, listings_queue: queue.Queue, ): self.high_low = high_low self.weapon_name = weapon_name self.listings_url_set = set() self.inspect_links_list = [] self.listings_info_list = [] self.listings_info_dict = {} self.best_or_worst_skins = [] self.webshare_ips = webshare_ips self.RequestManager = RequestManager(all_sessions, webshare_ips) self.listings_queue = listings_queue self.inspect_server_url = "http://23.88.122.57:1337/" def startBot(self): self.getHashedNameListHighOrLow() logger.info("1/2 Build Listing URLs finished (" + str(self.weapon_name) + ")") self.buildInspectLinksList() logger.info( "2/2 Build Inspect Links List finished (" + str(self.weapon_name) + ")" ) def getHashedNameListHighOrLow(self): end = False count = 0 repeat_counter = 0 if self.high_low == "high": wear = 4 elif self.high_low == "low": wear = 0 else: wear = 0 logger.critical("wear is not high or low, defaulting to low") if self.weapon_name == "gloves": all_skins_url = ( f"https://steamcommunity.com/market/search/render?currency=3&norender=1&query=&" "start={count}&count=10&search_descriptions=0&sort_column=price&sort_dir=asc&" "appid=730&category_730_ItemSet%5B%5D=any&category_730_ProPlayer%5B%5D=any&" "category_730_StickerCapsule%5B%5D=any&category_730_TournamentTeam%5B%5D=any&" "category_730_Weapon%5B%5D=any&category_730_Exterior%5B%5D=tag_WearCategory{wear}&" "category_730_Rarity%5B%5D=tag_Rarity_Ancient" ).format(count=count, wear=wear) else: all_skins_url = ( f"https://steamcommunity.com/market/search/render?currency=3&norender=1&query=&" "start={count}&count=10&search_descriptions=0&sort_column=price&sort_dir=asc&appid=730&category_730_ItemSet" "%5B%5D=any&category_730_ProPlayer%5B%5D=any&category_730_StickerCapsule%5B%5D=any" "&category_730_TournamentTeam%5B%5D=any&category_730_Weapon%5B%5D={weapon_name}" "&category_730_Exterior%5B%5D=tag_WearCategory{wear}" ).format(count=count, weapon_name=self.weapon_name, wear=wear) while not end: if self.weapon_name == "gloves": all_skins_url = ( f"https://steamcommunity.com/market/search/render?currency=3&norender=1&query=&" "start={count}&count=10&search_descriptions=0&sort_column=price&sort_dir=asc&" "appid=730&category_730_ItemSet%5B%5D=any&category_730_ProPlayer%5B%5D=any&" "category_730_StickerCapsule%5B%5D=any&category_730_TournamentTeam%5B%5D=any&" "category_730_Weapon%5B%5D=any&category_730_Exterior%5B%5D=tag_WearCategory{wear}&" "category_730_Rarity%5B%5D=tag_Rarity_Ancient" ).format(count=count, wear=wear) else: all_skins_url = ( f"https://steamcommunity.com/market/search/render?currency=3&norender=1&query=&" "start={count}&count=10&search_descriptions=0&sort_column=price&sort_dir=asc&appid=730&category_730_ItemSet" "%5B%5D=any&category_730_ProPlayer%5B%5D=any&category_730_StickerCapsule%5B%5D=any" "&category_730_TournamentTeam%5B%5D=any&category_730_Weapon%5B%5D={weapon_name}" "&category_730_Exterior%5B%5D=tag_WearCategory{wear}" ).format(count=count, weapon_name=self.weapon_name, wear=wear) #logger.critical("ALL SKINS URL: " + str(all_skins_url)) try: response = self.RequestManager.getRequestAllAvailableProxies( all_skins_url, skins_or_inspect_links="skins" ) if response is None: time.sleep(5) continue if isinstance(response, str): json_page = json.loads(response.text) else: json_page = response.json() # type: ignore except Exception as e: logger.critical(str(e)) continue repeat_counter += 1 if repeat_counter >= 2: random_time = random.uniform(0.5, 1.5) time.sleep(repeat_counter * random_time) if repeat_counter > 20: end = True logger.critical("json_page failed 20 times, skipping") continue if json_page == None: continue if json_page["start"] + 10 == json_page["total_count"]: end = True logger.critical("We end because of: " + str(int(json_page["start"]) + 10) + " == " + str(json_page["total_count"])) if json_page["start"] + 10 >= json_page["total_count"]: end = True logger.critical("We end because of: " + str(int(json_page["start"]) + 10) + " > " + str(json_page["total_count"])) else: logger.warning(str(json_page["start"]) + " - " + str(json_page['total_count'])) if len(json_page["results"]) == 0: continue for i in json_page["results"]: if ( "AK-47 | Aphrodite" in i["name"] or "M4A1-S | Party Animal" in i["name"] or "AWP | Exothermic" in i["name"] or "USP-S | Sleeping Potion" in i["name"] or "Zeus x27 | Earth Mandala" in i["name"] or "UMP-45 | Warm Blooded" in i["name"] or "Galil AR | Sky Mandala" in i["name"] or "Five-SeveN | Fraise Crane" in i["name"] or "XM1014 | XoooM" in i["name"] or "Tec-9 | Banana Leaf" in i["name"] or "SSG 08 | Calligrafaux" in i["name"] or "MP7 | Coral Paisley" in i["name"] or "MP5-SD | Picnic" in i["name"] or "Sawed-Off | Crimson Batik" in i["name"] or "SG 553 | Safari Print" in i["name"] or "R8 Revolver | Mauve Aside" in i["name"] or "PP-Bizon | Thermal Currents" in i["name"] or "MP9 | Bee-Tron" in i["name"] or "FAMAS | Byproduct" in i["name"] or "CZ75-Auto | Honey Paisley" in i["name"] or "AWP | The End" in i["name"] or "AK-47 | Breakthrough" in i["name"] or "Glock-18 | Trace Lock" in i["name"] or "AUG | Creep" in i["name"] or "Desert Eagle | The Daily Deagle" in i["name"] or "P2000 | Grip Tape" in i["name"] or "P90 | Aeolian Light" in i["name"] or "FAMAS | Vendetta" in i["name"] or "M4A4 | Aeolian Dark" in i["name"] or "MAC-10 | Snow Splash" in i["name"] or "MP5-SD | Snow Splash" in i["name"] or "R8 Revolver | Dark Chamber" in i["name"] or "PP-Bizon | Bizoom" in i["name"] or "Dual Berettas | Silver Pour" in i["name"] or "M249 | Sleet" in i["name"] or "MP9 | Dizzy" in i["name"] or "Nova | Currents" in i["name"] or "P250 | Sleet" in i["name"] or "SCAR-20 | Zinc" in i["name"] or "SSG 08 | Sans Comic" in i["name"] ): logger.info("SKIPPING NEW SKIN: " + str(i["name"])) continue if i["sell_listings"] == 0: logger.critical("NO LISTINGS FOR " + str(i["name"])) continue listings_url = ( "https://steamcommunity.com/market/listings/730/" + urllib.parse.quote(str(i["name"]).encode("utf-8")) + "/render/?query=" ) self.listings_url_set.add(listings_url) count += 10 if self.high_low == "high": with open("capped_skins/skins_high_new.json") as json_file_high: skins_high = json.load(json_file_high) if self.weapon_name in skins_high: for skin in skins_high[self.weapon_name]: full_skin_name = str(skin[0]) + " (" + str(skin[2]) + ")" listings_url = ( "https://steamcommunity.com/market/listings/730/" + urllib.parse.quote(str(full_skin_name).encode("utf-8")) + "/render/?query=" ) self.listings_url_set.add(listings_url) if self.high_low == "low": with open("capped_skins/skins_low_new.json") as json_file_low: skins_low = json.load(json_file_low) if self.weapon_name in skins_low: for skin in skins_low[self.weapon_name]: full_skin_name = str(skin[0]) + " (" + str(skin[1]) + ")" listings_url = ( "https://steamcommunity.com/market/listings/730/" + urllib.parse.quote(str(full_skin_name).encode("utf-8")) + "/render/?query=" ) self.listings_url_set.add(listings_url) def buildInspectLinksList(self): time.sleep(random.uniform(0.2, 0.5)) for listings_url in self.listings_url_set: if listings_url == "" or listings_url == None or listings_url == "\n": logger.critical("listings_url IS EMPTY????") continue end = False repeat_counter_overall = 0 start = 0 check_steamwebapi = True while not end: single_listings_url = ( str(listings_url) + "&start=" + str(start) + "&count=100¤cy=3" ) repeat_counter_overall += 1 if repeat_counter_overall > 1: random_time = random.uniform(1, 4) time.sleep(repeat_counter_overall + random_time) if repeat_counter_overall > 3 and check_steamwebapi is True: logger.error("repeat_counter_overall > 3") market_hash_name = str(listings_url).split("/")[6] steamwebapi_url = f"https://www.steamwebapi.com/steam/api/item?key=CJN62T0XGKR7TH5A&market_hash_name={market_hash_name}¤cy=EUR&with_groups=false" logger.error( "TRYING TO CHECK IF THERE ARE OFFERS FOR " + str(market_hash_name) + " WITH " + str(steamwebapi_url) ) try: response = self.RequestManager.getRequestNaked( steamwebapi_url ).json() if "offervolume" in response: if ( response["offervolume"] == 0 or response["offervolume"] is None ): logger.error( "NO OFFERS FOR " + str(market_hash_name) ) end = True if response["sold7d"] == 0: logger.error( "ALSO NO SELLS FOR 7 DAYS, ADDING TO EXCLUDED SKINS FOR " + str(market_hash_name) ) with open("excluded_skins.txt", "r") as file: content = file.read() excluded_skins = ast.literal_eval(content) excluded_skins.append(market_hash_name) with open("excluded_skins.txt", "w") as file: file.write(str(excluded_skins)) continue else: check_steamwebapi = False logger.error("THERE ARE OFFERS FOR " + str(market_hash_name) + " - SLEEPING, GOOD NIGHT!!!") time.sleep(5) else: if "status" in response: check_steamwebapi = False logger.error("STEAMWEBAPI FREE REQUESTS USED UP??" + str(response)) except Exception as e: logger.critical(str(e)) time.sleep(2) continue if repeat_counter_overall > 15: end = True logger.critical( "listings_page_json failed 15 times with " + str(single_listings_url) ) encoded_market_name = urllib.parse.urlparse( single_listings_url ).path.split("/")[4] if HelperFunctions.checkIfSkinExcluded(encoded_market_name) is True: end = True continue listings_page_response = self.getInspectLinks(single_listings_url) listings_page_json = listings_page_response.json() if listings_page_response is not None else None # type: ignore listings_page_text = listings_page_response.text if listings_page_response is not None else None # type: ignore if listings_page_json == None: time.sleep(2) continue if "listinginfo" not in listings_page_json: time.sleep(2) continue if "results_html" not in listings_page_json: time.sleep(2) continue if listings_page_json["listinginfo"] is None: time.sleep(2) continue if listings_page_json["results_html"] is None: time.sleep(2) continue end = True try: new_inspect_links = HelperFunctions.getNewInspectLinks(listings_page_text, high_low=self.high_low) # type: ignore if(len(new_inspect_links) == 0): logger.critical("NO INSPECT LINKS FOR " + str(encoded_market_name)) continue self.listings_queue.put(new_inspect_links) except Exception as e: logger.critical(str(e)) continue return None #new_inspect_links_list = [listingid, assetid, Price, inspectlink, highlow] def checkForRankNew(self, new_inspect_links_list): # inspect_link_dict = {"links": []} # results = [] # for link_info in new_inspect_links_list: # inspect_link = link_info[3] # inspect_link_dict["links"].append(inspect_link) for link_info in new_inspect_links_list: #check if assetid already in DB possibly_item_in_db = HelperFunctions.checkIfItemInDB(link_info[1]) if possibly_item_in_db is not None: if possibly_item_in_db[3] <= 5: # type: ignore self.csfloatRequest(link_info[3], new_inspect_links_list) else: self.csfloatRequest(link_info[3], new_inspect_links_list) # links_dict = {"links": []} # for link_info in new_inspect_links_list: # inspect_link = link_info[3] # links_dict["links"].append({"link": inspect_link}) # end = False # sorted_dict_top5 = {} # while not end: # try: # response = self.RequestManager.postRequestNaked(self.inspect_server_url + "/bulk", _json=links_dict).json() # except Exception as e: # logger.critical(str(e)) # time.sleep(2) # continue # end = True # try: # if self.high_low == "high": # sorted_dict = dict( # sorted( # response.items(), # key=lambda item: item[1]['floatvalue'], # reverse=True # ) # ) # else: # sorted_dict = dict( # sorted( # response.items(), # key=lambda item: item[1]['floatvalue'] # ) # ) # except Exception as e: # logger.critical(str(e)) # return None # sorted_dict_top5 = dict(list(sorted_dict.items())[:5]) # if sorted_dict_top5 is not None and len(sorted_dict_top5) > 0: # for item in sorted_dict_top5: # single_iteminfo = sorted_dict_top5[item] # top5_in_db = False # for link_info in new_inspect_links_list: # check_assetid = link_info[1] # listing_id = link_info[0] # price = link_info[2] # if str(check_assetid) == str(single_iteminfo["a"]): # inspect_link = link_info[3] # full_item_name = single_iteminfo["full_item_name"] if "full_item_name" in single_iteminfo else None # if full_item_name is None: # logger.critical("full_item_name is None for inspect link: " + str(inspect_link)) # break # single_iteminfo_db_entry = HelperFunctions.checkIfItemInDB(single_iteminfo["a"]) # if single_iteminfo_db_entry is not None: # if single_iteminfo_db_entry[3] > 5: # type: ignore # logger.info(f"Inspect link {inspect_link} with float {single_iteminfo['floatvalue']} is higher than existing DB entry with float {single_iteminfo_db_entry[5]}, skipping potential buy for this listing") # type: ignore # break # else: # top5_in_db = True # else: # pass # #self.generateFinalListing(single_iteminfo, inspect_link, price, listing_id, rank=1337) # break # for iteminfo in response: # #sort all iteminfo by response["iteminfo"]["floatvalue"] # results.append({ # "floatvalue": iteminfo["floatvalue"], # "rankvalue": iteminfo.get("rankvalue"), # "iteminfo": iteminfo # }) # results.sort(key=lambda x: x["floatvalue"], reverse=(self.high_low == "high")) # if sorted_dict_top5 is not None and len(sorted_dict_top5) > 0: # for item in sorted_dict_top5: # single_iteminfo = sorted_dict_top5[item] # for link_info in new_inspect_links_list: # check_assetid = link_info[1] # listing_id = link_info[0] # price = link_info[2] # if str(check_assetid) == str(single_iteminfo["a"]): # inspect_link = link_info[3] # with topfloatdb.db_cursor() as cur: # cur.execute("SELECT * FROM ownskinsdb WHERE a = %s", (str(single_iteminfo["a"]),)) # result = cur.fetchone() # if result is not None: # logger.warning(f"AssetID {single_iteminfo["a"]} already in DB, skipping rank check for this inspect link: {link_info[3]}") # continue # else: # # try: # # response = self.RequestManager.getRequestNaked( # # self.inspect_server_url + "?url=" + inspect_link # # ).json() # # if response is not None: # # end = True # # #break # # except Exception as e: # # logger.critical(str(e)) # # time.sleep(2) # # continue # full_item_name = single_iteminfo["full_item_name"] if "full_item_name" in single_iteminfo else None # if full_item_name is None: # logger.critical("full_item_name is None for inspect link: " + str(inspect_link)) # continue # #get all db entries with same full_item_name # with topfloatdb.db_cursor() as cur: # if self.high_low == "high": # cur.execute("SELECT * FROM ownskinsdb WHERE full_item_name = %s ORDER BY floaty DESC LIMIT 5", (str(full_item_name),)) # else: # cur.execute("SELECT * FROM ownskinsdb WHERE full_item_name = %s ORDER BY floaty ASC LIMIT 5", (str(full_item_name),)) # results = cur.fetchall() # rank_counter = 1 # top5 = False # if results is not None and len(results) > 0: # for result in results: # #print(result) # #check if response["iteminfo"]["floatvalue"] is in the top 5 # if "floatvalue" in single_iteminfo: # floatvalue = single_iteminfo["floatvalue"] # if self.high_low == "high": # if floatvalue >= result[5]: # type: ignore # logger.info(f"Inspect link {inspect_link} with float {floatvalue} is higher than existing DB entry with float {result[5]}, skipping potential buy for this listing") # top5 = True # break # else: # if floatvalue <= result[5]: # type: ignore # logger.info(f"Inspect link {inspect_link} with float {floatvalue} is lower than existing DB entry with float {result[5]}, skipping potential buy for this listing") # top5 = True # break # rank_counter += 1 # if top5 is True or results is None or len(results) == 0 or len(results) < 5: # self.csfloatRequest(inspect_link, new_inspect_links_list) # #HelperFunctions.insertItemIntoDB(full_item_name=str(response["iteminfo"]["full_item_name"]), inspect_link=str(inspect_link), iteminfo=str(response["iteminfo"]), rank=int(rank), a=int(assetid), floaty=str(response["iteminfo"]["floatvalue"])) # end = False # while not end: # for link_info in new_inspect_links_list: # #check if assetid already in DB # assetid = link_info[1] # inspect_link = link_info[3] def csfloatRequest(self, inspect_link, new_inspect_links_list): end = False floatdb_counter = 0 while not end: try: response_raw = self.RequestManager.postRequestEvomi( "https://gateway.floatdb.com/v1/ranks/check", data={"links": [inspect_link]} ) # type: ignore if response_raw is None: floatdb_counter += 1 time.sleep(2) continue if floatdb_counter > 5: end = True logger.critical("FloatDB request failed 5 times for inspect link: " + str(inspect_link)) continue response = response_raw.json() # type: ignore #logger.critical(str(response)) if "error" in response: floatdb_counter += 1 time.sleep(2) continue end = True except Exception as e: logger.critical(str(e)) floatdb_counter += 1 time.sleep(2) continue if "ranks" in response: # type: ignore ranks = response["ranks"] # type: ignore for assetid in ranks: rank_dict = ranks[assetid] key, rank = list(rank_dict.items())[0] for link_info in new_inspect_links_list: check_assetid = link_info[1] listing_id = link_info[0] price = link_info[2] if str(check_assetid) == str(assetid): inspect_link = link_info[3] end = False counter = 0 while not end: counter += 1 if counter > 5: end = True logger.critical("Failed to get iteminfo from inspect server after 5 tries for inspect link: " + str(inspect_link)) continue try: response = self.RequestManager.getRequestNaked( self.inspect_server_url + "?url=" + inspect_link ).json() if response is not None: end = True #break except Exception as e: logger.critical(str(e)) time.sleep(2) continue if int(rank) <= 5: logger.error(f"RANK CHECK RESPONSE FOR INSPECT LINK {inspect_link}: " + str(response)) self.generateFinalListing(response, inspect_link, price, listing_id, rank) try: with topfloatdb.db_cursor() as cur: cur.execute( "INSERT INTO ownskinsdb (full_item_name, inspect_link, iteminfo, rank, a, floaty) VALUES (%s, %s, %s, %s, %s, %s)", (str(response["iteminfo"]["full_item_name"]), str(inspect_link), str(response["iteminfo"]), int(rank), int(assetid), response["iteminfo"]["floatvalue"]), ) logger.critical("Rows affected: " + str(cur.rowcount)) logger.critical(f"Inserted item with inspect link {inspect_link} into DB with rank {rank} and float {response['iteminfo']['floatvalue']}") #HelperFunctions.insertItemIntoDB(full_item_name=, inspect_link=str(inspect_link), iteminfo=str(response["iteminfo"]), rank=int(rank), a=int(assetid), floaty=response["iteminfo"]["floatvalue"]) except psycopg2.errors.UniqueViolation as e: logger.warning(f"Item with inspect link {inspect_link} already exists in DB, skipping. Error: " + str(e)) continue except Exception as e: logger.critical(str(e)) continue break # try: # response = self.RequestManager.getRequestNaked( # self.inspect_server_url + "?url=" + inspect_link # ).json() # HelperFunctions.insertItemIntoDB(full_item_name=str(response["iteminfo"]["full_item_name"]), inspect_link=str(inspect_link), iteminfo=str(response["iteminfo"]), rank=int(rank), a=int(assetid), floaty=str(response["iteminfo"]["floatvalue"])) # else: # pass # else: # logger.error("NO RANKS IN RANK CHECK RESPONSE FOR " + str(inspect_link_dict) + ": " + str(response)) # except Exception as e: # logger.critical(str(e)) def getInspectLinks(self, single_listings_url: str): try: response = self.RequestManager.getRequestAllAvailableProxies( single_listings_url, False # type: ignore ) if response is None: return None return response except Exception as e: logger.critical(str(e)) return None def generateFinalListing(self, iteminfo, inspect_link, price, listing_id, rank): market_link = ( "https://steamcommunity.com/market/listings/730/" + str(iteminfo["iteminfo"]["full_item_name"]).replace(" ", "%20") + "#buylisting|" + str(listing_id) + "|730|2|" + str(iteminfo["iteminfo"]["a"]) ) final_listing = FinalListing(full_item_name=str(iteminfo["iteminfo"]["full_item_name"]).replace(" ", "%20"), market_url=market_link, inspect_link=str(inspect_link), rank=rank, price=str(price)) self.singleCheckForPotentialBuy(final_listing, rank) def singleCheckForPotentialBuy(self, final_listing: FinalListing, rank=0): single_cheap_top_skin = {} end = False repeat_counter = 0 while not end: lowest_price = 0 repeat_counter += 1 try: price_overview_page = self.RequestManager.getRequestAllAvailableProxies( "https://steamcommunity.com/market/priceoverview/?market_hash_name=" + HelperFunctions.encodeAidsItemName(final_listing.full_item_name) + "&appid=730¤cy=3", "price", ) if price_overview_page is None: if repeat_counter > 10: end = True logger.error( "Couldn't get price for " + str(final_listing.full_item_name) + " after 10 times, skipping for now" ) continue price_overview_json = json.loads(price_overview_page.text) # type: ignore except Exception as e: logger.critical(str(e)) continue if price_overview_json == None: continue if "lowest_price" in price_overview_json: lowest_price = price_overview_json["lowest_price"] lowest_price = lowest_price[:-1] lowest_price = str(lowest_price).replace(",", ".") else: response = self.RequestManager.getRequestAllAvailableProxies( "https://steamcommunity.com/market/listings/730/" + str(final_listing.full_item_name), "price", ) if price_overview_page is None: continue listing_source_page = response.text # type: ignore nameid_url = HelperFunctions.manuallyGetNameIDURL( listing_source_page=listing_source_page ) if nameid_url is None: continue else: alt_response = self.RequestManager.getRequestAllAvailableProxies( nameid_url, "price" ) if alt_response is None: continue alt_price_overview_json = json.loads(alt_response.text) # type: ignore if len(str(alt_price_overview_json["lowest_sell_order"])) > 2: lowest_price = str( alt_price_overview_json["lowest_sell_order"] )[:-2] lowest_price = str(lowest_price).replace(" ", "") lowest_price = str(lowest_price).replace("--", "0") try: if float(final_listing.price) <= float(lowest_price) * 5: single_cheap_top_skin[urllib.parse.unquote(final_listing.full_item_name)] = [ str(final_listing.rank), str(final_listing.price), str(final_listing.market_url), ] HelperFunctions.writeSingleCheapSkinToFile(single_cheap_top_skin) else: if final_listing.inspect_link != "" and rank == 1337: try: postgresql_conn = psycopg2.connect( database="postgres", user="postgres", password="Berufsorientierung1!", host="23.88.122.57", port="6432", ) postgresql_cur = postgresql_conn.cursor() postgresql_cur.execute( "INSERT INTO floats(inspect_link, rank) VALUES(%s, %s)", ( final_listing.inspect_link, rank, ), ) postgresql_conn.commit() postgresql_cur.close() except (Exception, psycopg2.DatabaseError) as error: logger.error(str(error)) finally: if postgresql_conn is not None: # type: ignore postgresql_conn.close() # type: ignore except ValueError: logger.critical("ValueError in singleCheckForPotentialBuy: " + str(e)) # type: ignore end = True