ازگر کا استعمال کرتے ہوئے واقعی ملازمت کی فہرستوں کو کھرچنے کا طریقہ

pip install playwright

pip install lxml

pip install pandas

playwright install

import asyncio
from playwright.async_api import async_playwright

async def get_page_content(url):
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=False,
            proxy = {
                'server': '',
                'username': '',
                'password': ''
            }
        )  # Headed browser
        page = await browser.new_page()
        await page.goto(url)
        
        # Extract the page's content
        content = await page.content()
        
        await browser.close()  # Close the browser once done
        return content

from lxml import html

def parse_job_listings(content):
    # Parse HTML content
    parser = html.fromstring(content)
    
    # Extract each job posting using XPath
    job_posting = parser.xpath('//ul[@class="css-zu9cdh eu4oa1w0"]/li')
    
    jobs_data = []
    for element in job_posting[:-1]:  # Skip the last element if it's an ad or irrelevant
        title = ''.join(element.xpath('.//h2/a/span/@title'))
        if title:
            link = ''.join(element.xpath('.//h2/a/@href'))
            location = ''.join(element.xpath('.//div[@data-testid="text-location"]/text()'))
            description = ', '.join(element.xpath('.//div[@class="css-9446fg eu4oa1w0"]/ul//li/text()'))
            company_name = ''.join(element.xpath('.//span[@data-testid="company-name"]/text()'))

            # Append extracted data to the jobs_data list
            jobs_data.append({
                'Title': title,
                'Link': f"https://www.indeed.com{link}",
                'Location': location,
                'Description': description,
                'Company': company_name
            })
    
    return jobs_data

import pandas as pd

async def scrape_indeed_jobs(url):
    # Step 1: Get page content using Playwright
    content = await get_page_content(url)
    
    # Step 2: Parse the HTML and extract job details
    jobs_data = parse_job_listings(content)
    
    return jobs_data

# URL to scrape
url = 'https://www.indeed.com/q-usa-jobs.html'

# Scraping and saving data
async def main():
    # Scrape job data from the specified URL
    jobs = await scrape_indeed_jobs(url)
    
    # Step 3: Save data to CSV using pandas
    df = pd.DataFrame(jobs)
    df.to_csv('indeed_jobs.csv', index=False)
    
    print("Data saved to indeed_jobs.csv")

# Run the main function
asyncio.run(main())

async def scrape_multiple_pages(base_url, pages=3):
    all_jobs = []
    
    for page_num in range(pages):
        # Update URL for pagination
        url = f"{base_url}&start={page_num * 10}"
        print(f"Scraping page: {url}")
        
        # Scrape job data from each page
        jobs = await scrape_indeed_jobs(url)
        all_jobs.extend(jobs)
    
    # Save all jobs to CSV
    df = pd.DataFrame(all_jobs)
    df.to_csv('indeed_jobs_all_pages.csv', index=False)
    print("Data saved to indeed_jobs_all_pages.csv")

# Scrape multiple pages of job listings
asyncio.run(scrape_multiple_pages('https://www.indeed.com/jobs?q=usa', pages=3))

query = "python+developer"
base_url = f"https://www.indeed.com/jobs?q={query}"
asyncio.run(scrape_multiple_pages(base_url, pages=3))

import asyncio
from playwright.async_api import async_playwright
from lxml import html
import pandas as pd

# Step 1: Fetch page content using Playwright
async def get_page_content(url):
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=False
            proxy = {
                'server': '',
                'username': '',
                'password': ''
            }
        )  # Run browser in headed mode
        page = await browser.new_page()
        await page.goto(url, wait_until='networkidle')
        
        # Extract page content
        content = await page.content()
        await browser.close()  # Close browser after use
        return content

# Step 2: Parse the HTML content using lxml
def parse_job_listings(content):
    # Parse the HTML using lxml
    parser = html.fromstring(content)
    
    # Select individual job postings using XPath
    job_posting = parser.xpath('//ul[@class="css-zu9cdh eu4oa1w0"]/li')
    
    # Extract job data
    jobs_data = []
    for element in job_posting[:-1]:
        title = ''.join(element.xpath('.//h2/a/span/@title'))
        if title:
            link = ''.join(element.xpath('.//h2/a/@href'))
            location = ''.join(element.xpath('.//div[@data-testid="text-location"]/text()'))
            description = ', '.join(element.xpath('.//div[@class="css-9446fg eu4oa1w0"]/ul//li/text()'))
            company_name = ''.join(element.xpath('.//span[@data-testid="company-name"]/text()'))

            # Append extracted data to the jobs_data list
            jobs_data.append({
                'Title': title,
                'Link': f"https://www.indeed.com{link}",
                'Location': location,
                'Description': description,
                'Company': company_name
            })
    
    return jobs_data

# Step 3: Scrape Indeed jobs for a single page
async def scrape_indeed_jobs(url):
    # Get page content using Playwright
    content = await get_page_content(url)
    
    # Parse HTML and extract job data
    jobs_data = parse_job_listings(content)
    
    return jobs_data

# Step 4: Handle pagination and scrape multiple pages
async def scrape_multiple_pages(base_url, query, pages=3):
    all_jobs = []
    
    for page_num in range(pages):
        # Update the URL to handle pagination and add the search query
        url = f"{base_url}?q={query}&start={page_num * 10}"
        print(f"Scraping page: {url}")
        
        # Scrape jobs for the current page
        jobs = await scrape_indeed_jobs(url)
        all_jobs.extend(jobs)
    
    # Save all jobs to a CSV file
    df = pd.DataFrame(all_jobs)
    df.to_csv(f'indeed_jobs_{query}.csv', index=False)
    print(f"Data saved to indeed_jobs_{query}.csv")

# Function to run the scraper with dynamic query input
async def run_scraper():
    # Step 5: Ask user for input query and number of pages to scrape
    query = input("Enter the job title or keywords to search (e.g., python+developer): ")
    pages = int(input("Enter the number of pages to scrape: "))
    
    # Scrape jobs across multiple pages based on the query
    base_url = 'https://www.indeed.com/jobs'
    await scrape_multiple_pages(base_url, query, pages)

# Run the scraper
asyncio.run(run_scraper())