智慧

引言

我在家的這段時間嘗試了一下Cursor，給個具體的例子好了。

例子

事實上我一行程式都沒寫，我只是不斷提出修改要求，以及標識bug所在。下面是我要求它寫的一個Pillowfort爬蟲

import requests
from bs4 import BeautifulSoup
import os
import time
from urllib.parse import urljoin
import hashlib
import json
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException
import re
import urllib.parse

class PillowfortScraper:
    def __init__(self):
        self.session = requests.Session()
        self.base_url = 'https://www.pillowfort.social'
        self.download_history_file = 'download_history.json'
        self.download_history = self.load_download_history(self.download_history_file)
        # Initialize Selenium WebDriver
        options = webdriver.ChromeOptions()
        options.add_argument('--headless')  # Run in headless mode
        self.driver = webdriver.Chrome(options=options)
        self.default_tag = 'bakertoons'
        
    def __del__(self):
        """Cleanup when the scraper is destroyed"""
        if hasattr(self, 'driver'):
            self.driver.quit()

    def load_download_history(self, history_file):
        """Load download history from JSON file"""
        try:
            with open(history_file, 'r') as f:
                return json.load(f)
        except FileNotFoundError:
            return {'urls': {}, 'downloaded': {}}
            
    def save_download_history(self, history_file, history):
        """Save download history to JSON file"""
        with open(history_file, 'w') as f:
            json.dump(history, f)
            
    def get_url_hash(self, url):
        """Generate a hash for the URL"""
        return hashlib.md5(url.encode()).hexdigest()

    def login(self, email, password):
        """Login to Pillowfort account"""
        login_url = urljoin(self.base_url, '/login')
        
        print("Attempting to log in...")
        
        # First get the authenticity token
        try:
            response = self.session.get(login_url)
            soup = BeautifulSoup(response.text, 'html.parser')
            auth_token = soup.find('input', {'name': 'authenticity_token'})['value']
            
            login_data = {
                'user[email]': email,
                'user[password]': password,
                'authenticity_token': auth_token
            }
            
            response = self.session.post(login_url, data=login_data)
            
            if response.ok:
                print("Successfully logged in!")
                return True
            else:
                print(f"Login failed. Status code: {response.status_code}")
                return False
                
        except Exception as e:
            print(f"Login error: {str(e)}")
            return False

    def get_original_image_url(self, url):
        """Convert thumbnail URL to original size URL"""
        # Example URL conversion:
        # From: https://img3.pillowfort.social/posts/thumbnail_...jpg
        # To:   https://img3.pillowfort.social/posts/original_...jpg
        return url.replace('thumbnail_', 'original_')

    def scroll_to_bottom(self):
        """Scroll to bottom of page to trigger lazy loading"""
        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)  # Wait for content to load

    def has_next_page(self):
        """Check if there are more posts to load"""
        try:
            # Check for next page button using specific XPATH
            next_button = self.driver.find_elements(By.XPATH, '//*[@id="userBlogPosts"]/div/dir-pagination-controls/ul/li[last()]/a')
            if next_button and 'disabled' not in next_button[0].find_element(By.XPATH, '..').get_attribute('class'):
                return True
            return False
        except NoSuchElementException:
            return False

    def get_user_posts(self, username, tag=None):
        try:
            # Use default tag if none provided
            tag = tag or self.default_tag
            # URL encode the tag for special characters
            encoded_tag = urllib.parse.quote(tag)
            url = f"{self.base_url}/{username}/tagged/{encoded_tag}"
            
            print(f"\nStarting scrape from: {url}")
            self.driver.get(url)
            
            post_data = []
            page = 1
            total_images = 0
            
            while True:
                print(f"\nScraping page {page}...")
                
                try:
                    print("Waiting for posts to load...")
                    # Wait for Angular to finish rendering
                    WebDriverWait(self.driver, 10).until(
                        EC.presence_of_element_located((By.CLASS_NAME, "post"))
                    )
                except TimeoutException:
                    print("Timeout waiting for posts to load")
                    break

                # Let Angular finish updating the DOM
                time.sleep(2)

                # Parse current page
                soup = BeautifulSoup(self.driver.page_source, 'html.parser')
                posts = soup.find_all("div", class_="post")
                print(f"Found {len(posts)} posts on page {page}")
                
                # Process posts
                page_images = 0
                for post_idx, post in enumerate(posts, 1):
                    content = post.find("div", class_="post-content")
                    if not content:
                        continue
                        
                    images = []
                    for img in content.find_all("img"):
                        src = img.get("src")
                        if src and not src.endswith(".svg"):
                            original_src = src.replace("_small", "")
                            filename = self.get_image_filename(original_src)
                            images.append({
                                'url': original_src,
                                'filename': filename
                            })
                            page_images += 1
                    
                    if images:
                        post_data.append({
                            "images": images
                        })
                
                total_images += page_images
                print(f"Found {page_images} images on page {page}")
                print(f"Total images found so far: {total_images}")
                
                # Try to go to next page
                print("\nLooking for next page...")
                if not self.click_next_page():
                    print("No more pages available")
                    break
                
                page += 1
                print("Waiting for next page to load...")
                time.sleep(2)  # Increased wait time for page load
            
            print(f"\nScraping complete!")
            print(f"Total pages scraped: {page}")
            print(f"Total posts with images: {len(post_data)}")
            print(f"Total images found: {total_images}")
            return post_data

        except Exception as e:
            print(f"\nError during scraping: {str(e)}")
            return []

    def download_images(self, username, save_dir, tag=None):
        """Download images with original filenames and handle history"""
        try:
            # Use default tag if none provided
            tag = tag or self.default_tag
            
            # Create nested directory structure: ./username/tag
            save_dir = os.path.join(save_dir, username, tag)
            history_file = os.path.join(save_dir, 'download_history.json')
            
            if not os.path.exists(save_dir):
                os.makedirs(save_dir)
                print(f"\nCreated directory: {save_dir}")
            
            # Load download history
            history = self.load_download_history(history_file)
            
            print("\nStarting scraping process...")
            print("Press Ctrl+C to exit gracefully")
            posts = []
            page = 1
            found_existing = False
            
            while True:
                print(f"\nScraping page {page}...")
                current_posts = self.get_user_posts(username, tag)
                
                # Process posts and check for existing URLs
                for post in current_posts:
                    new_images = []
                    for image in post['images']:
                        url = image['url']
                        url_hash = self.get_url_hash(url)
                        
                        # Check if URL is already in history and was successfully downloaded
                        if url_hash in history['urls'] and history['downloaded'].get(url_hash, False):
                            found_existing = True
                            continue
                        
                        # Add to new images if not in history or previous download failed
                        new_images.append(image)
                        history['urls'][url_hash] = {
                            'url': url,
                            'filename': image['filename'],
                            'timestamp': time.time()
                        }
                    
                    if new_images:
                        posts.append({'images': new_images})
                
                # If we found existing URLs and no new images to download, stop pagination
                if found_existing and not any(post['images'] for post in posts):
                    print("Reached previously downloaded content, stopping...")
                    break
                
                # Try to go to next page
                if not self.click_next_page():
                    print("No more pages available")
                    break
                
                page += 1
            
            if not posts:
                print("No new images to download")
                return
            
            total_images = sum(len(post['images']) for post in posts)
            print(f"\nStarting download of {total_images} new images...")
            print(f"Files will be saved to: {save_dir}")
            
            downloaded = 0
            failed = 0
            for post_idx, post in enumerate(posts, 1):
                for image in post['images']:
                    try:
                        url = image['url']
                        url_hash = self.get_url_hash(url)
                        filename = image['filename']
                        save_path = os.path.join(save_dir, filename)
                        
                        # Skip if file already exists and is marked as downloaded
                        if os.path.exists(save_path) and history['downloaded'].get(url_hash, False):
                            print(f"Skipping already downloaded: {filename}")
                            continue
                        
                        print(f"Downloading: {filename}")
                        
                        # Use timeout for requests
                        response = requests.get(url, timeout=10)
                        if response.status_code == 200:
                            with open(save_path, 'wb') as f:
                                f.write(response.content)
                            downloaded += 1
                            history['downloaded'][url_hash] = True
                            print(f"✓ Success ({downloaded}/{total_images})")
                        else:
                            failed += 1
                            history['downloaded'][url_hash] = False
                            print(f"✗ Skipped: HTTP {response.status_code}")
                            continue
                            
                    except requests.Timeout:
                        failed += 1
                        history['downloaded'][url_hash] = False
                        print(f"✗ Skipped: Download timeout")
                        continue
                    except Exception as e:
                        failed += 1
                        history['downloaded'][url_hash] = False
                        print(f"✗ Skipped: {str(e)}")
                        continue
                    
                    # Save history periodically (every 5 successful downloads)
                    if downloaded % 5 == 0:
                        self.save_download_history(history_file, history)
                        print(f"\nProgress: {downloaded}/{total_images} downloaded, {failed} skipped")
            
            # Final save of history
            self.save_download_history(history_file, history)
                    
            print(f"\nDownload complete!")
            print(f"Successfully downloaded: {downloaded} new images")
            print(f"Skipped downloads: {failed} images")
            print(f"Files saved in: {save_dir}")

        except KeyboardInterrupt:
            print("\n\nGracefully exiting...")
            print("Saving download history...")
            self.save_download_history(history_file, history)
            print(f"Progress saved: {downloaded}/{total_images} downloaded")
            print("You can resume the download later")
            return

    def get_image_filename(self, url):
        """Extract original filename from URL"""
        # Parse the URL and get the path
        path = urllib.parse.urlparse(url).path
        # Get the base filename
        base_name = os.path.basename(path)
        # Remove _small suffix if present
        base_name = base_name.replace('_small', '')
        return base_name

    def click_next_page(self):
        """Click the next page button if available"""
        try:
            # Find the next page button using XPATH
            next_button = self.driver.find_element(By.XPATH, '//*[@id="userBlogPosts"]/div/dir-pagination-controls/ul/li[last()]/a')
            
            # Check if parent li has disabled class
            parent_li = next_button.find_element(By.XPATH, '..')
            if 'disabled' in parent_li.get_attribute('class'):
                print("Reached last page")
                return False
            
            # Scroll the button into view
            self.driver.execute_script("arguments[0].scrollIntoView(true);", next_button)
            time.sleep(1)  # Give time for any animations to complete
            
            print("Found next page button, clicking...")
            # Try JavaScript click if regular click fails
            try:
                next_button.click()
            except:
                self.driver.execute_script("arguments[0].click();", next_button)
            
            # Wait for Angular to update the page
            time.sleep(2)  
            return True
            
        except NoSuchElementException:
            print("No next page button found")
            return False
        except Exception as e:
            print(f"Error clicking next page: {str(e)}")
            return False

    def get_user_posts_with_pagination(self, username, tag=None, max_pages=None):
        """Get all posts from a user's blog, optionally filtered by tag, with pagination handling"""
        posts = []
        page = 1
        
        while True:
            print(f"Scraping page {page}...")
            
            # Get posts for the current page
            current_posts = self.get_user_posts(username, tag)
            if not current_posts:
                print("No more posts to load")
                break
            
            posts.extend(current_posts)
            
            # Try to go to next page
            if not self.click_next_page():
                print("No more pages to load")
                break
            
            page += 1
            time.sleep(1)  # Be nice to the server
        
        print(f"Found {len(posts)} posts with images")
        return posts

    def download_images_with_pagination(self, save_dir, username, tag=None):
        """Download images with original filenames and handle pagination"""
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)
            
        posts = self.get_user_posts_with_pagination(username, tag)
        total_images = sum(len(self.extract_images(post)) for post in posts)
        print(f"Starting download of {total_images} images...")
        
        downloaded = 0
        for post in posts:
            images = self.extract_images([post])
            for image in images:
                try:
                    filename = self.get_image_filename(image)
                    save_path = os.path.join(save_dir, filename)
                    
                    # Download the image
                    response = self.session.get(image)
                    if response.status_code == 200:
                        with open(save_path, 'wb') as f:
                            f.write(response.content)
                        downloaded += 1
                        print(f"Downloaded {downloaded}/{total_images}: {filename}")
                    else:
                        print(f"Failed to download {filename}: HTTP {response.status_code}")
                        
                except Exception as e:
                    print(f"Error downloading {filename}: {str(e)}")
                    
        print(f"Download complete. Successfully downloaded {downloaded} images.")

def main():
    try:
        # Initialize scraper
        scraper = PillowfortScraper()
        
        # Default settings
        default_email = "YOUR EMAIL"
        default_password = "YOUR PASSWORD"
        default_username = "bakertoons"
        default_save_dir = os.getcwd()  # current working directory
        
        print("Press Ctrl+C at any time to exit gracefully")
        
        # Get credentials with defaults
        email = input(f"Enter your Pillowfort email [{default_email}]: ").strip()
        email = email if email else default_email
        
        password = input(f"Enter your Pillowfort password [{default_password}]: ").strip()
        password = password if password else default_password
        
        # Get target username with default
        username = input(f"Enter the username to scrape [{default_username}]: ").strip()
        username = username if username else default_username
        
        # Get tag (handle spaces)
        tag = input(f"Enter tag to search for [{scraper.default_tag}]: ").strip()
        tag = tag if tag else scraper.default_tag
        
        # Use current directory as base, will create username/tag subdirectories
        save_dir = default_save_dir
        
        # Login
        if not scraper.login(email, password):
            print("Exiting due to login failure.")
            return
        
        # Download images
        print("\nStarting download process...")
        scraper.download_images(username, save_dir, tag)

    except KeyboardInterrupt:
        print("\nExiting program...")
    finally:
        if hasattr(scraper, 'driver'):
            scraper.driver.quit()
        print("Goodbye!")

if __name__ == "__main__":
    main()

已經測試過，在macOS下是可以正確運行。

令人驚嘆嗎，也許是，但又不完全是，後面是我的一些思考。

反思

要說我一程式碼行沒寫，也不完全是這樣。我寫過靜態網站的爬蟲，但從來沒有處理過動態網站。Cursor最初給的程式bug連連，但我注意到其中一處錯誤，並猜測用XPath解決尋找下一頁的問題，這樣才正確地運行起來。

這是一個很好的例子，中間包含我明白以及不明白的地方，讓我思考使用這類工具的邊界在哪裡。我花了不少時間去測試才得出一個正確的結果。我認同日後的編程範式要改變了，正確使用AI可以幫你省下很多力氣，其實在我看來閱讀API文檔以及調試也都是沒有什麼技術含量的工作。這不由得引申出另一個問題，AI適合幹什麼事情？

就像機械運動適合重複工作一樣，和古早時期的語音助手比起來，至少能幫你改寫程式碼，翻譯文章這一塊，就已經讓人驚嘆了。這是因為現在的AI引入了Self Attention的緣故，可以連繫上下文的關聯來決定輸出。我不想在這裡討論個中細節，我直接給出結論，AI適合去講「正確的廢話」。

這也是AI文本輸出很像人的緣故，我相信它應該很容易通過圖靈測試，因為它就是那樣被訓練的。你可以反思嬰兒學習說話的過程，也包括你自己，你真的思考過你說出來的話中每個詞的意思嗎，我想應該不是，它們像是被打散的積木又被重新組裝了起來。你之所以這樣說話，以及你之所以用到這個詞，完全是因為你以前聽過，或者見到別人這樣用過。而這正是AI的訓練方式，AI像是閱讀了大量文檔，或者文獻的助手，它不會給你任何有價值的信息，或者表面現象的深層連繫。因此它需要你不斷提出具體的要求，這正是它缺乏自我意識與演繹推理的緣故。

因此我可以總結，它適合從事沒有創造力的工作，這一點它做得最為出色。比如一些報告和呈文，信息量越低，它做得越出色。因此AI適合翻譯，潤色以及生成模板。光譜的另一端是創造性的思考，這一點我沒看出當前的AI有這樣的能力。

從這個角度來看，當前的AI並不是什麼真正的智慧。它頂多像是模仿鳥類的風箏。如果要造出飛機，那也是瞭解空氣動力學以後的事情。

我不想去預測AI的未來，給一個不確定性未來以一個確定的結果是世界上最愚蠢的事情。不過一個有趣的現象是你掌握的語言限制了你的思考能力，類似維特根斯坦說語言是思維的邊界。AI的流行在某種程度上也反映了大多數人在日常並不習慣於思考，或者從事的僅僅是機械性的重複勞動。我想洞見事物內在的本質聯繫，或許是有自我意識生命的特權。