Python کے ساتھ Walmart کا ڈیٹا اسکریپ کرنے کی گائیڈ

pip install requests
pip install  lxml
pip install urllib3

import requests
from lxml import html
import csv
import random
import urllib3
import ssl

product_urls = [
    'link with https',
    'link with https',
    'link with https'
]

user_agents = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
]

proxy = [
    '<ip>:<port>',
    '<ip>:<port>',
    '<ip>:<port>',
]

headers = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'accept-language': 'en-IN,en;q=0.9',
    'dnt': '1',
    'priority': 'u=0, i',
    'sec-ch-ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Linux"',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'none',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
}

product_details = []

for url in product_urls:
   headers['user-agent'] = random.choice(user_agents)
   proxies = {
       'http': f'http://{random.choice(proxy)}',
       'https': f'http://{random.choice(proxy)}',
   }
   try:
       # Send an HTTP GET request to the URL
       response = requests.get(url=url, headers=headers, proxies=proxies, verify=False)
       print(response.status_code)
       response.raise_for_status()
   except requests.exceptions.RequestException as e:
       print(f'Error fetching data: {e}')

   # Parse the HTML content using lxml
   parser = html.fromstring(response.text)
   # Extract product title
   title = ''.join(parser.xpath('//h1[@id="main-title"]/text()'))
   # Extract product price
   price = ''.join(parser.xpath('//span[@itemprop="price"]/text()'))
   # Extract review details
   review_details = ''.join(parser.xpath('//div[@data-testid="reviews-and-ratings"]/div/span[@class="w_iUH7"]/text()'))

   # Store extracted details in a dictionary
   product_detail = {
       'title': title,
       'price': price,
       'review_details': review_details
   }
   # Append product details to the list
   product_details.append(product_detail)

with open('walmart_products.csv', 'w', newline='') as csvfile:
    fieldnames = ['title', 'price', 'review_details']
    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
    writer.writeheader()
    for product_detail in product_details:
        writer.writerow(product_detail)

import requests
from lxml import html
import csv
import random
import urllib3
import ssl

ssl._create_default_https_context = ssl._create_stdlib_context
urllib3.disable_warnings()


# List of product URLs to scrape Walmart data
product_urls = [
   'link with https',
   'link with https',
   'link with https'
]

# Randomized User-Agent strings for anonymity
user_agents = [
   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
   'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
   'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
]

# Proxy list for IP rotation
proxy = [
    '<ip>:<port>',
    '<ip>:<port>',
    '<ip>:<port>',
]


# Headers to mimic browser requests
headers = {
   'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
   'accept-language': 'en-IN,en;q=0.9',
   'dnt': '1',
   'priority': 'u=0, i',
   'sec-ch-ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
   'sec-ch-ua-mobile': '?0',
   'sec-ch-ua-platform': '"Linux"',
   'sec-fetch-dest': 'document',
   'sec-fetch-mode': 'navigate',
   'sec-fetch-site': 'none',
   'sec-fetch-user': '?1',
   'upgrade-insecure-requests': '1',
}

# Initialize an empty list to store product details
product_details = []

# Loop through each product URL
for url in product_urls:
   headers['user-agent'] = random.choice(user_agents)
   proxies = {
       'http': f'http://{random.choice(proxy)}',
       'https': f'http://{random.choice(proxy)}',
   }
   try:
       # Send an HTTP GET request to the URL
       response = requests.get(url=url, headers=headers, proxies=proxies, verify=False)
       print(response.status_code)
       response.raise_for_status()
   except requests.exceptions.RequestException as e:
       print(f'Error fetching data: {e}')

   # Parse the HTML content using lxml
   parser = html.fromstring(response.text)
   # Extract product title
   title = ''.join(parser.xpath('//h1[@id="main-title"]/text()'))
   # Extract product price
   price = ''.join(parser.xpath('//span[@itemprop="price"]/text()'))
   # Extract review details
   review_details = ''.join(parser.xpath('//div[@data-testid="reviews-and-ratings"]/div/span[@class="w_iUH7"]/text()'))

   # Store extracted details in a dictionary
   product_detail = {
       'title': title,
       'price': price,
       'review_details': review_details
   }
   # Append product details to the list
   product_details.append(product_detail)

# Write the extracted data to a CSV file
with open('walmart_products.csv', 'w', newline='') as csvfile:
   fieldnames = ['title', 'price', 'review_details']
   writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
   writer.writeheader()
   for product_detail in product_details:
       writer.writerow(product_detail)