ازگر کے ساتھ متحرک ویب سائٹوں کو کھرچنے کے لئے رہنما

pip install playwright
pip install lxml

playwright install

import asyncio
from playwright.async_api import async_playwright

async def scrape_instagram():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)  # Headless mode No visual feedback
        page = await browser.new_page()
        
        # Visit the profile URL
        await page.goto("https://www.instagram.com/profile name/", wait_until="networkidle")

        # Click the button to load more posts 
        await page.get_by_role("button", name="Show more posts from").click()
        
        # Scroll the page to load dynamic content
        scroll_count = 5  # Customize this based on how many times you want to scroll
        for _ in range(scroll_count):
            await page.evaluate('window.scrollBy(0, 700);')
            await page.wait_for_timeout(3000)  # Wait for posts to load
            await page.wait_for_load_state("networkidle")
        
        # Get the page content
        content = await page.content()
        await browser.close()
        
        return content

# Run the asynchronous function
asyncio.run(scrape_instagram())

from lxml import html
import json

def extract_post_urls(page_content):
    # Parse the HTML content using lxml
    tree = html.fromstring(page_content)
    
    # XPath for extracting post URLs
    post_urls_xpath = '//a[contains(@href, "/p/")]/@href'
    
    # Extract URLs
    post_urls = tree.xpath(post_urls_xpath)
    
    # Convert relative URLs to absolute
    base_url = "https://www.instagram.com"
    post_urls = [f"{base_url}{url}" for url in post_urls]
    
    return post_urls

def save_data(profile_url, post_urls):
    data = {profile_url: post_urls}
    with open('instagram_posts.json', 'w') as json_file:
        json.dump(data, json_file, indent=4)

# Scrape and extract URLs
page_content = asyncio.run(scrape_instagram())
post_urls = extract_post_urls(page_content)

# Save the extracted URLs in a JSON file
save_data("https://www.instagram.com/profile name/", post_urls)

(window.scrollBy(0, 700))

 wait_for_load_state("networkidle")

async def scrape_with_proxy():
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=False, 
            proxy={"server": "http://your-proxy-server:port"}
        )
        page = await browser.new_page()
        await page.goto("https://www.instagram.com/profile name/", wait_until="networkidle")
        # Continue scraping as before...

async def scrape_with_proxy():
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=False, 
            proxy={"server": "http://your-proxy-server:port", "username": "username", "password": "password"}
        )
        page = await browser.new_page()
        await page.goto("https://www.instagram.com/profile name/", wait_until="networkidle")
        # Continue scraping as before...

import asyncio
from playwright.async_api import async_playwright
from lxml import html
import json

# Function to automate browser and scrape dynamic content with proxies
async def scrape_instagram(profile_url, proxy=None):
    async with async_playwright() as p:
        # Set up browser with proxy if provided
        browser_options = {
            'headless': True,  # Use headed browser to see the action (can set to True for headless mode)
        }
        if proxy:
            browser_options['proxy'] = proxy

        # Launch the browser
        browser = await p.chromium.launch(**browser_options)
        page = await browser.new_page()

        # Visit the Instagram profile page
        await page.goto(profile_url, wait_until="networkidle")
        
        # Try clicking the "Show more posts" button (optional, might fail if button not found)
        try:
           await page.click('button:has-text("Show more posts from")')
        except Exception as e:
           print(f"No 'Show more posts' button found: {e}")


        # Scroll the page to load more posts
        scroll_count = 5  # Number of scrolls to load posts
        for _ in range(scroll_count):
            await page.evaluate('window.scrollBy(0, 500);')
            await page.wait_for_timeout(3000)  # Wait for new posts to load
            await page.wait_for_load_state("networkidle")

        # Get the complete page content after scrolling
        content = await page.content()
        await browser.close()  # Close the browser once done
        
        return content

# Function to parse the scraped page content and extract post URLs
def extract_post_urls(page_content):
    # Parse the HTML content using lxml
    tree = html.fromstring(page_content)
    
    # XPath for extracting post URLs
    post_urls_xpath = '//a[contains(@href, "/p/")]/@href'
    
    # Extract post URLs using the XPath
    post_urls = tree.xpath(post_urls_xpath)
    
    # Convert relative URLs to absolute URLs
    base_url = "https://www.instagram.com"
    post_urls = [f"{base_url}{url}" for url in post_urls]
    
    return post_urls

# Function to save the extracted post URLs into a JSON file
def save_data(profile_url, post_urls):
    # Structure the data in JSON format
    data = {profile_url: post_urls}
    
    # Save the data to a file
    with open('instagram_posts.json', 'w') as json_file:
        json.dump(data, json_file, indent=4)
    print(f"Data saved to instagram_posts.json")

# Main function to run the scraper and save the data
async def main():
    # Define the Instagram profile URL
    profile_url = "https://www.instagram.com/profile name/"
    
    # Optionally, set up a proxy
    proxy = {"server": "server", "username": "username", "password": "password"}  # Use None if no proxy is required
    
    # Scrape the Instagram page with proxies
    page_content = await scrape_instagram(profile_url, proxy=proxy)
    
    # Extract post URLs from the scraped page content
    post_urls = extract_post_urls(page_content)
    
    # Save the extracted post URLs into a JSON file
    save_data(profile_url, post_urls)

if __name__ == '__main__':
   asyncio.run(main())