Delete output

output/scan_result_20241030_210745.txt

@@ -1,242 +0,0 @@
-#ファイルパス
-Get_URL_list/get_url_list.py
-------------
-import json
-import requests
-from bs4 import BeautifulSoup
-
-# Load URLs from JSON file
-with open('ideabte_scraping/Get_URL_list/URL_json_output/debate_urls.json', 'r') as f:
-    json_urls = json.load(f)
-
-# Function to get sub-page URLs from a main theme URL
-def get_debate_topic_urls(main_url):
-    response = requests.get(main_url)
-    soup = BeautifulSoup(response.text, 'html.parser')
-    
-    # Extract all links from the main URL page
-    links = soup.find_all('a', href=True)
-    
-    # Filter for links that are debate topics
-    topic_urls = [link['href'] for link in links if link['href'].startswith('/')]
-    
-    # Make URLs absolute
-    full_urls = [f"https://idebate.net{url}" for url in topic_urls if "~b" in url]
-    
-    return full_urls
-
-# Dictionary to store all debate topic URLs for each main theme URL
-all_debate_topic_urls = {}
-for theme_url in json_urls:
-    theme_name = theme_url.split("/")[-2].replace("~", "_")
-    all_debate_topic_urls[theme_name] = get_debate_topic_urls(theme_url)
-
-# Output the results
-with open('ideabte_scraping/Get_URL_list/output/debate_topic_urls.json', 'w') as f:
-    json.dump(all_debate_topic_urls, f, indent=4)
-
-print("Debate topic URLs have been saved to debate_topic_urls.json")
-
-#ファイルパス
-scraping_idebate/run_main.sh
-------------
-#!/bin/bash
-
-# Set default paths
-JSON_FILE="ideabte_scraping/Get_URL_list/output/debate_topic_urls.json"
-OUTPUT_DIR="ideabte_scraping/scraping_idebate/output"
-
-# Check if the JSON file exists
-if [ ! -f "$JSON_FILE" ]; then
-    echo "Error: JSON file '$JSON_FILE' does not exist."
-    exit 1
-fi
-
-# Create the output directory if it doesn't exist
-mkdir -p "$OUTPUT_DIR"
-
-# Run the Python script
-python3 ideabte_scraping/scraping_idebate/src/scraping.py "$JSON_FILE" "$OUTPUT_DIR"
-
-echo "Scraping completed. Output files are stored in $OUTPUT_DIR"
-
-#ファイルパス
-scraping_idebate/src/scraping.py
-------------
-import requests
-from bs4 import BeautifulSoup
-import json
-import os
-import sys
-from urllib.parse import urlparse
-
-def scrape_url(url, output_dir):
-    response = requests.get(url)
-    response.raise_for_status()
-
-    soup = BeautifulSoup(response.content, 'html.parser')
-    topic = soup.find("h1", class_="blog-post__title").get_text(strip=True)
-
-    points_list = []
-
-    def extract_points(section, section_name):
-        accordion_items = section.find_next_sibling('div', class_='accordion').find_all('div', class_='accordion__item')
-        for item in accordion_items:
-            point_subtitle = item.find('h4', class_='accordion__subtitle').get_text().strip()
-            point_body = item.find('div', class_='accordion__body').find('p').get_text().strip()
-            points_list.append({
-                "topic": topic,
-                "section": section_name,
-                "context": f"**{point_subtitle}**\n{point_body}"
-            })
-
-    points_for_section = soup.find('div', class_='points-vote points-vote--for')
-    if points_for_section:
-        extract_points(points_for_section, "Points For")
-
-    points_against_section = soup.find('div', class_='points-vote points-vote--against')
-    if points_against_section:
-        extract_points(points_against_section, "Points Against")
-
-    # Generate a unique filename based on the URL
-    parsed_url = urlparse(url)
-    filename = f"{parsed_url.path.strip('/').replace('/', '_')}.json"
-    output_path = os.path.join(output_dir, filename)
-
-    with open(output_path, "w", encoding="utf-8") as f:
-        json.dump(points_list, f, ensure_ascii=False, indent=4)
-
-    print(f"Data saved to {output_path}")
-
-if __name__ == "__main__":
-    if len(sys.argv) != 3:
-        print("Usage: python script.py <json_file> <output_dir>")
-        sys.exit(1)
-
-    json_file = sys.argv[1]
-    output_dir = sys.argv[2]
-
-    os.makedirs(output_dir, exist_ok=True)
-
-    with open(json_file, 'r') as f:
-        url_data = json.load(f)
-
-    for category, urls in url_data.items():
-        for url in urls:
-            try:
-                scrape_url(url, output_dir)
-            except Exception as e:
-                print(f"Error scraping {url}: {str(e)}")
-
-#ファイルパス
-scraping_idebate/src/scraping_test.py
-------------
-import requests
-from bs4 import BeautifulSoup
-
-url = "https://idebate.net/this-house-would-make-all-museums-free-of-charge~b641/"
-
-# ウェブページを取得
-response = requests.get(url)
-response.raise_for_status()  # エラーチェック
-
-# HTMLを解析
-soup = BeautifulSoup(response.content, 'html.parser')
-
-# Points Forのdiv要素を取得
-points_for_section = soup.find('div', class_='points-vote points-vote--for')
-
-# ポイントを含むアコーディオン要素を取得
-accordion_items = points_for_section.find_next_sibling('div', class_='accordion').find_all('div', class_='accordion__item')
-
-# 各ポイントのテキストを抽出
-points = []
-for item in accordion_items:
-    point_subtitle = item.find('h4', class_='accordion__subtitle').get_text().strip()
-    point_body = item.find('div', class_='accordion__body').find('p').get_text().strip()
-    points.append(f"**{point_subtitle}**\n{point_body}")
-
-# 抽出したポイントを出力
-for point in points:
-    print(point)
-    print("-" * 20)  # 区切り線
-
-
-#ファイルパス
-scraping_idebate/src/scraping_tqdm.py
-------------
-import requests
-from bs4 import BeautifulSoup
-import json
-import os
-import sys
-from urllib.parse import urlparse
-from tqdm import tqdm
-
-def scrape_url(url, output_dir):
-    response = requests.get(url)
-    response.raise_for_status()
-
-    soup = BeautifulSoup(response.content, 'html.parser')
-    topic = soup.find("h1", class_="blog-post__title").get_text(strip=True)
-
-    points_list = []
-
-    def extract_points(section, section_name):
-        accordion_items = section.find_next_sibling('div', class_='accordion').find_all('div', class_='accordion__item')
-        for item in accordion_items:
-            point_subtitle = item.find('h4', class_='accordion__subtitle').get_text().strip()
-            point_body = item.find('div', class_='accordion__body').find('p').get_text().strip()
-            points_list.append({
-                "topic": topic,
-                "section": section_name,
-                "context": f"**{point_subtitle}**\n{point_body}"
-            })
-
-    points_for_section = soup.find('div', class_='points-vote points-vote--for')
-    if points_for_section:
-        extract_points(points_for_section, "Points For")
-
-    points_against_section = soup.find('div', class_='points-vote points-vote--against')
-    if points_against_section:
-        extract_points(points_against_section, "Points Against")
-
-    # Generate a unique filename based on the URL
-    parsed_url = urlparse(url)
-    filename = f"{parsed_url.path.strip('/').replace('/', '_')}.json"
-    output_path = os.path.join(output_dir, filename)
-
-    with open(output_path, "w", encoding="utf-8") as f:
-        json.dump(points_list, f, ensure_ascii=False, indent=4)
-
-    return output_path
-
-if __name__ == "__main__":
-    if len(sys.argv) != 3:
-        print("Usage: python script.py <json_file> <output_dir>")
-        sys.exit(1)
-
-    json_file = sys.argv[1]
-    output_dir = sys.argv[2]
-
-    os.makedirs(output_dir, exist_ok=True)
-
-    with open(json_file, 'r') as f:
-        url_data = json.load(f)
-
-    total_urls = sum(len(urls) for urls in url_data.values())
-    
-    with tqdm(total=total_urls, desc="Scraping Progress") as pbar:
-        for category, urls in url_data.items():
-            for url in urls:
-                try:
-                    output_path = scrape_url(url, output_dir)
-                    pbar.set_postfix_str(f"Saved: {output_path}")
-                    pbar.update(1)
-                except Exception as e:
-                    pbar.set_postfix_str(f"Error: {url}")
-                    print(f"\nError scraping {url}: {str(e)}")
-                    pbar.update(1)
-
-    print("\nScraping completed. All data saved to the output directory.")
-