# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# # Predicting top news articles

# # 1. Importing libraries

import newspaper
from newspaper import Config
from newspaper import Article
import uuid
import pandas as pd
import time
import requests
import pprint
from bs4 import BeautifulSoup

# +
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10


# -

def get_date_range(start_date,end_date):
    from datetime import date,timedelta
    start_date=date(int(start_date.split('-')[0]),int(start_date.split('-')[1]),int(start_date.split('-')[2]))
    end_date=date(int(end_date.split('-')[0]),int(end_date.split('-')[1]),int(end_date.split('-')[2]))
    return [str(start_date+timedelta(i)).replace('-','') for i in range((end_date-start_date).days+1)]


# # 2. Retrieve ABC News articles

dates=get_date_range('2019-08-03','2019-09-02')+get_date_range('2019-10-04','2019-11-03')
news_url='http://abcnews.go.com'

abc_trends=[]
abc_news=[]
subsites=['us','international','politics','entertainment','technology','health','lifestyle','business']
for date in dates:
    #Trending articles
    req=requests.get('https://web.archive.org/web/'+str(date)+'1/'+news_url)
    soup=BeautifulSoup(req.text,'html.parser')
    abc_date_trends=[n.findChild('a')['href'].split("https://")[2] for n in soup.findAll('li',{'class':'headlines-li story'})]
    abc_date_trends+=[n.findChild('a')['href'].split("https://")[2] for n in soup.findAll('li',{'class':'headlines-li wirestory'})]
    abc_date_trends=list(set(abc_date_trends))
    print(date,'trends',len(abc_date_trends))
    #Regular news
    abc_date_news=[]
    for subsite in subsites:
        req=requests.get('https://web.archive.org/web/'+str(date)+'1/'+news_url+'/'+subsite)
        soup=BeautifulSoup(req.text,'html.parser')
        abc_date_news+=[n.findChild('a')['href'].split("https://")[2] for n in soup.findAll('li',{'class':'headlines-li story'})]
        abc_date_news+=[n.findChild('a')['href'].split("https://")[2] for n in soup.findAll('li',{'class':'headlines-li wirestory'})]
        abc_date_news+=[n.findChild('a',{'class':'black-ln'})['href'].split("https://")[2] for n in soup.findAll('li',{'class':'headlines-li actual'}) if n.findChild('a',{'class':'black-ln'}) is not None and len(n.findChild('a',{'class':'black-ln'})['href'].split("https://"))>2]
        abc_date_news=list(set(abc_date_news))
        abc_date_news=[n for n in abc_date_news if n not in abc_date_trends]
        print(date,subsite,len(abc_date_news))
    abc_news+=abc_date_news
    abc_trends+=abc_date_trends
    abc_news=list(set(abc_news))
    abc_trends=list(set(abc_trends))
    abc_news=[n for n in abc_news if n not in abc_trends]
    print(date,'total news:',len(abc_news),'total trends:',len(abc_trends))

abc_news=list({a.split('-')[-1]:a for a in abc_news}.values())
abc_trends=list({a.split('-')[-1]:a for a in abc_trends}.values())
abc_news=[a for a in abc_news if a not in abc_trends]



# +
import pickle
with open('abc_regular_news.pickle', 'wb') as handle:
    pickle.dump(abc_news, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('abc_top_news.pickle', 'wb') as handle:
    pickle.dump(abc_trends, handle, protocol=pickle.HIGHEST_PROTOCOL)


# -

with open('abc_regular_news.pickle', 'rb') as handle:
    abc_news_bkp = pickle.load(handle)
with open('abc_top_news.pickle', 'rb') as handle:
    abc_trends_bkp = pickle.load(handle)



# # 3. Retrieve WSJ trending news

dates=get_date_range('2019-08-03','2019-09-02')+get_date_range('2019-10-04','2019-11-03')
news_url='https://www.wsj.com'

wsj_trending_news_urls=[]
for date in dates:
    req=requests.get('https://web.archive.org/web/'+str(date)+'1/'+news_url)
    soup=BeautifulSoup(req.text,'html.parser')
    daily_top_news=list(set([a['href'] for a in soup.findAll('a') if 'href' in a.attrs and '?mod=hp_lead_pos' in a['href'] and not '#comments' in a['href'] and not 'mod=article_inline' in a['href'] ]))
    wsj_trending_news_urls+=daily_top_news
    wsj_trending_news_urls=list(set(wsj_trending_news_urls))
    print(date, 'today:', len(daily_top_news), 'total:', len(wsj_trending_news_urls))

#remove wayback machine preamble
wsj_trending_news_urls=list(set([a.split("https://")[2] for a in wsj_trending_news_urls]))

#remove query strings from url
wsj_trending_news_urls=list(set([a.split('?')[0] for a in wsj_trending_news_urls]))

#extract and remove duplicate ids
wsj_trending_news_urls=list({a.split('-')[-1]:a for a in wsj_trending_news_urls}.values())

import pickle
with open('wsj_top_news.pickle', 'wb') as handle:
    pickle.dump(wsj_trending_news_urls, handle, protocol=pickle.HIGHEST_PROTOCOL)

len(wsj_trending_news_urls)

news_url+'/news/archive/2019/11/10/'

# +
articles=[]
from selenium import webdriver
news_url='https://www.wsj.com'
option = webdriver.ChromeOptions()
chrome_prefs = {}
option.experimental_options["prefs"] = chrome_prefs
chrome_prefs["profile.default_content_settings"] = {"images": 2}
chrome_prefs["profile.managed_default_content_settings"] = {"images": 2}
read_driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=option)

dates=get_date_range('2019-08-03','2019-09-02')+get_date_range('2019-10-04','2019-11-03')
for date in dates:
    date_url=date[0:4]+'/'+date[4:6]+'/'+date[6:8]
    read_driver.get(news_url+'/news/archive/'+date_url+'?page=1')
    soup=BeautifulSoup(read_driver.page_source, 'html.parser')
    try:
        pagecount=int(soup.find('span',{'class':"WSJTheme--pagepicker-total--Kl350I1l"}).text.strip()[-2:].strip())
    except:
        pagecount=1
    articles+=list(set([a['href'] for a in soup.find('main',{'id':'main'}).findChildren('a') if 'href' in a.attrs and 'wsj.com' in a['href']]))
    if pagecount>1:
        for page in range(2,pagecount+1):
            read_driver.get(news_url+'/news/archive/'+date_url+'?page='+str(page))
            soup=BeautifulSoup(read_driver.page_source, 'html.parser') 
            articles+=list(set([a['href'] for a in soup.find('main',{'id':'main'}).findChildren('a') if 'href' in a.attrs and 'wsj.com' in a['href']]))
    articles=list(set(articles))
    print(date)
# -

articles=list(set(articles))
len(articles)

wsj_regular_news=list({a.split('-')[-1]:a for a in articles}.values())
len(wsj_regular_news)

wsj_regular_news=[n for n in wsj_regular_news if n not in wsj_trending_news_urls]
len(wsj_regular_news)



with open('wsj_regular_news.pickle', 'wb') as handle:
    pickle.dump(wsj_regular_news_urls, handle, protocol=pickle.HIGHEST_PROTOCOL)

# # 4. Retrieve Irish Times news

irish_trending_articles=[]
irish_regular_articles=[]
import re
dates=get_date_range('2019-08-03','2019-09-02')+get_date_range('2019-10-04','2019-11-03')
for date in dates:
    url='https://www.irishtimes.com/'
    soup=BeautifulSoup(requests.get('https://web.archive.org/web/'+date+'1/'+url).text,'html.parser')
    irish_trending_articles+=list(set([url+a['href'].strip()[1:] for a in soup.findAll('a') if 'data-evt-label' in a.attrs and 'href' in a.attrs and 'Main Story Headline' in a['data-evt-label']]))
    soup=BeautifulSoup(requests.get(url+'static/articleindex/'+date[0:4]+'/'+date[0:4]+'-'+date[4:6]+'-'+date[6:8]+'.html').text,'html.parser')
    irish_regular_articles+=list(set([a['href'] for a in soup.find('ul',{'class':'article-list'}).findAll('a') if 'href' in a.attrs and re.search(r'[0-9]',a['href'][-1]) and 'https://' not in a['href'] and 'opinion' not in a['href']]))
    print(date,'trending total:',len(irish_trending_articles),'total:',len(irish_regular_articles))

irish_trending_articles=list(set(irish_trending_articles))
irish_regular_articles=list(set(irish_regular_articles))
print('trending total:',len(irish_trending_articles),'total:',len(irish_regular_articles))

irish_trending_articles_list=list(set(list({a.split("-")[-1]:a for a in irish_trending_articles}.values())))
irish_regular_articles_list=list(set(list({a.split("-")[-1]:a for a in irish_regular_articles}.values())))
print('trending total:',len(irish_trending_articles_list),'total:',len(irish_regular_articles_list))

irish_regular_articles_list=[url+n[1:] for n in irish_regular_articles_list if n not in irish_trending_articles_list]
print('trending total:',len(irish_trending_articles_list),'total:',len(irish_regular_articles_list))

with open('irish_regular_news.pickle', 'wb') as handle:
    pickle.dump(irish_regular_articles_list, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('irish_trending_news.pickle', 'wb') as handle:
    pickle.dump(irish_trending_articles_list, handle, protocol=pickle.HIGHEST_PROTOCOL)

# # 5. Retrieve Reuters news

url='https://www.reuters.com'
feeds_url='https://feeds.reuters.com/reuters/topnews'
reuters_news_regular=[]
reuters_news_trending=[]

dates=get_date_range('2019-08-03','2019-09-02')+get_date_range('2019-10-04','2019-11-03')
subsites=['finance','markets','world','politics','technology','wealth','lifestyle']
for date in dates:
    try:
        req=requests.get('https://web.archive.org/web/'+str(date)+'/'+feeds_url)
        soup=BeautifulSoup(req.text,'html.parser')
        trend_add=list(set([a.text.split('?')[0].split('https://')[1] for a in soup.findAll('guid') if url in a.text and '-id' in a.text]))
        reuters_news_trending+=trend_add
        reuters_news_trending=list(set(list({a.split('-')[-1]:a for a in reuters_news_trending}.values())))
        if len(trend_add)<1:
            continue
        req=requests.get('https://web.archive.org/web/'+str(date)+'/'+url)
        soup=BeautifulSoup(req.text,'html.parser')
        reuters_news_regular+=list(set([a['href'].split('?')[0].split('https://')[1] for a in soup.findAll('a') if 'href' in a.attrs and url in a['href'] and '-id' in a['href']]))
        for subsite in subsites:
            req=requests.get('https://web.archive.org/web/'+str(date)+'/'+url+'/'+subsite)
            soup=BeautifulSoup(req.text,'html.parser')
            reuters_news_regular+=list(set([a['href'].split('?')[0].split('https://')[1] for a in soup.findAll('a') if 'href' in a.attrs and url in a['href'] and '-id' in a['href']]))
            print(date,subsite)
        reuters_news_regular=list(set(list({a.split('-')[-1]:a for a in reuters_news_regular}.values())))
        reuters_news_regular=[n for n in reuters_news_regular if n not in reuters_news_trending]
        print(date,'trending total:',len(reuters_news_trending),'total:',len(reuters_news_regular))
    except:
        print(date,'had error')
        continue

reuters_news_regular=list(set(reuters_news_regular))
reuters_news_trending=list(set(reuters_news_trending))

len(reuters_news_trending)

len(reuters_news_regular)

import pickle
with open('reuters_regular_news.pickle', 'wb') as handle:
    pickle.dump(reuters_news_regular, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('reuters_trending_news.pickle', 'wb') as handle:
    pickle.dump(reuters_news_trending, handle, protocol=pickle.HIGHEST_PROTOCOL)

# # 6. Get article features

import newspaper
from newspaper import Config
from newspaper import Article
import pandas as pd
import pickle
from bs4 import BeautifulSoup

#1. ABC
with open('abc_regular_news.pickle', 'rb') as handle:
    abc_regular = pickle.load(handle)
with open('abc_top_news.pickle', 'rb') as handle:
    abc_top = pickle.load(handle)
print(abc_regular[0:2],abc_top[0:2])
#id filtering on regular and checks of unique ids
abc_top_ids=[n.split('-')[-1] for n in abc_top]
abc_regular=[n for n in abc_regular if n.split('-')[-1] not in abc_top_ids]
abc_regular_ids=[n.split('-')[-1] for n in abc_regular]
print(len([i for i in abc_regular_ids if i in abc_top_ids]))  #check if any id in regular appear on top. Should give 0
print(len(abc_regular),len(abc_regular_ids),len(abc_top),len(abc_top_ids)) #check if we have 1 article per id. Pairs should be equal

#2. WSJ
with open('wsj_top_news.pickle', 'rb') as handle:
    wsj_top = pickle.load(handle)
with open('wsj_regular_news.pickle', 'rb') as handle:
    wsj_regular = pickle.load(handle)
print(wsj_regular[0:2],wsj_top[0:2])
#id filtering on regular and checks of unique ids
wsj_top_ids=[n.split('-')[-1] for n in wsj_top]
wsj_regular=[n for n in wsj_regular if n.split('-')[-1] not in wsj_top_ids]
wsj_regular_ids=[n.split('-')[-1] for n in wsj_regular]
print(len([i for i in wsj_regular_ids if i in wsj_top_ids])) #check if any id in regular appear on top. Should give 0
print(len(wsj_regular),len(wsj_regular_ids),len(wsj_top),len(wsj_top_ids)) #check if we have 1 article per id. Pairs should be equal

#3. Irish
with open('irish_trending_news.pickle', 'rb') as handle:
    irish_top = pickle.load(handle)
with open('irish_regular_news.pickle', 'rb') as handle:
    irish_regular = pickle.load(handle)
print(irish_regular[0:2],irish_top[0:2])
#id filtering on regular and checks of unique ids
irish_top_ids=[n.split('-')[-1] for n in irish_top]
irish_regular=[n for n in irish_regular if n.split('-')[-1] not in irish_top_ids]
irish_regular_ids=[n.split('-')[-1] for n in irish_regular]
print(len([i for i in irish_regular_ids if i in irish_top_ids])) #check if any id in regular appear on top. Should give 0
print(len(irish_regular),len(irish_regular_ids),len(irish_top),len(irish_top_ids)) #check if we have 1 article per id. Pairs should be equal

#4. Reuters
with open('reuters_trending_news.pickle', 'rb') as handle:
    reuters_top = pickle.load(handle)
with open('reuters_regular_news.pickle', 'rb') as handle:
    reuters_regular = pickle.load(handle)
print(reuters_regular[0:2],reuters_top[0:2])
#id filtering on regular and checks of unique ids
reuters_top_ids=[n.split('-')[-1] for n in reuters_top]
reuters_regular=[n for n in reuters_regular if n.split('-')[-1] not in reuters_top_ids]
reuters_regular_ids=[n.split('-')[-1] for n in reuters_regular]
print(len([i for i in reuters_regular_ids if i in reuters_top_ids])) #check if any id in regular appear on top. Should give 0
print(len(reuters_regular),len(reuters_regular_ids),len(reuters_top),len(reuters_top_ids)) #check if we have 1 article per id. Pairs should be equal

#5. NYTimes
with open('nytimes_trending.pickle', 'rb') as handle:
    nytimes_top = pickle.load(handle)
with open('nytimes_all.pickle', 'rb') as handle:
    nytimes_regular = pickle.load(handle)
nytimes_top=list(set([n.split('http://')[1] for n in nytimes_top if len(n.split('http://'))>1]+[n.split('https://')[1] for n in nytimes_top if len(n.split('https://'))>1]))
nytimes_regular=list(set([n.split('https://')[1] for n in nytimes_regular]))
print(nytimes_regular[0:2],nytimes_top[0:2])
#id filtering irrelevant; check content equivalence
print(len(nytimes_regular))
nytimes_regular=[n for n in nytimes_regular if n not in nytimes_top]
print(len(nytimes_regular),len(nytimes_top)) #check if stories on top have been removed


#6. Newsweek
with open('newsweek_trending.pickle', 'rb') as handle:
    newsweek_top = pickle.load(handle)
with open('newsweek_all.pickle', 'rb') as handle:
    newsweek_regular = pickle.load(handle)
newsweek_top=list(set([n for n in newsweek_top if len(n.replace('www.newsweek.com','').split('-'))>3]))
newsweek_regular=list(set([n for n in newsweek_regular if len(n.replace('www.newsweek.com','').split('-'))>3]))
print(newsweek_regular[0:2],newsweek_top[0:2])
#id filtering on regular and checks of unique ids
newsweek_top_ids=[n.split('-')[-1] for n in newsweek_top]
newsweek_regular=[n for n in newsweek_regular if n.split('-')[-1] not in newsweek_top_ids]
newsweek_regular_ids=[n.split('-')[-1] for n in newsweek_regular]
print(len([i for i in newsweek_regular_ids if i in newsweek_top_ids])) #check if any id in regular appear on top. Should give 0
print(len(newsweek_regular),len(newsweek_regular_ids),len(newsweek_top),len(newsweek_top_ids)) #check if we have 1 article per id. Pairs should be equal

#Set user agent to parse
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15'
config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

source_names={'abc-news':'ABC News',
              'the-wall-street-journal':'The Wall Street Journal',
              'newsweek':'Newsweek',
              'reuters':'Reuters',
              'the-irish-times':'The Irish Times',
              'the-new-york-times':'The New York Times'}


def get_article_details(url_list,is_top=0):
    counter=0
    article_data=[]
    article_content=[]
    article_image_count=[]
    for url in url_list:
        url=url.strip()
        if not url.startswith('https'):
            url='https://'+url
        if 'photos-of-the-week' in url:
            continue
        #Infer source name from url
        if 'abcnews.' in url:
            source_id='abc-news'
        if '.newsweek.' in url:
            source_id='newsweek'
        if '.reuters.' in url:
            source_id='reuters'
        if '.irishtimes.' in url:
            source_id='the-irish-times'
        if '.nytimes.' in url:
            source_id='the-new-york-times'
        if '.wsj.' in url:
            source_id='the-wall-street-journal'
        #Download article
        article = Article(url, config=config)
        article.download()
        try:
            article.parse()
        except:
            continue
        #Author
        if source_id in ['abc-news']:
            try:
                author=article.authors[1]
            except:
                author=article.meta_data['author']
        elif source_id in ['the-wall-street-journal']:
            try:
                author=sorted([value for (key, value) in article.meta_data.items()if key == 'author'])[0]
            except:
                author='The Wall Street Journal'
        elif source_id in ['newsweek','reuters','the-new-york-times']:
            try:
                author=article.authors[0]
            except:
                author=source_names[source_id]
        elif source_id in ['the-irish-times']:
            body=requests.get(url.strip()).text
            soup=BeautifulSoup(body,'html.parser')
            try:
                author=soup.find('div',{'class':'author'}).text.strip().split(',')[0]
            except:
                try:
                    author=soup.find('div',{'class':'author'}).findChild('span').text.strip().split(',')[0]
                except:
                    author='The Irish Times'

        #Title
        if source_id in ['the-wall-street-journal']:
            title=[value for (key, value) in article.meta_data.items() if key == 'article.headline'][0]
        else:
            title=article.title
    
        #Description
        if source_id in ['the-wall-street-journal']:
            description=article.text.split('\n')[0]
            if description=='Resume Subscription':
                description=article.meta_data['article.summary']
        else:
            description=article.meta_data['description']

        #Published date   
        if source_id in ['the-wall-street-journal']:
            published_date=str([value for (key, value) in article.meta_data.items() if key == 'article.published'][0])
        elif source_id in ['abc-news']:
            published_date=article.meta_data['lastPublishedDate']
        elif source_id in ['newsweek','reuters']:
            published_date=str(article.publish_date)
        elif source_id in ['the-irish-times']:
            published_date=soup.find('time').text  
        elif source_id in ['the-new-york-times']:
            published_date=article.meta_data['article']['published_time']
    
        #Keyword
        if source_id in ['the-wall-street-journal','newsweek','the-irish-times','the-new-york-times']:
            try:
                keywords=''.join({value for (key, value) in article.meta_data.items() if key == 'news_keywords'})
                article_top_keyword = keywords.lower().split(',')[0]
            except:
                article_top_keyword=''
        elif source_id in ['abc-news','reuters']:
            try:
                article_top_keyword=article.meta_data['keywords'].lower().split(',')[0]
            except:
                article_top_keyword=''

        #content
        if source_id in ['reuters']:
            try:
                content=article.text[re.search(r'\s\w+\s\(Reuters\)',article.text).start()+1:]
            except:
                content=article.text
        elif source_id not in ['the-irish-times']:
            content=article.text
            if 'Resume Subscription' in content:
                content=''
        else:
            try:
                content=soup.find('div',{'class':'article_bodycopy'}).text.strip()
            except:
                try:
                    content=soup.find('div',{'class':'article_bodycopy_sample'}).text.strip()
                except:
                    content=''
        if content=='':
            content_text=''
        else:
            content_text=content[0:260].replace('\n',' ').strip()+' … [+'+str(len(content)-260)+' chars]'
        article_data_add={'article-id':str(uuid.uuid4()),
         'source-id':source_id,
         'source_name':source_names[source_id],
         'author':author,
         'title':title,
         'description':description,
         'url':url.strip(),
         'url_to_image':article.top_image,
         'published_at':published_date,
         'content':content_text,
         'top_keyword':article_top_keyword,
         'top_article':is_top}

        article_content_add={'article-id':article_data_add['article-id'],
                 'url':url.strip(),
                 'full_content':content}
        article_image_count_add={'article-id':article_data_add['article-id'],
                 'url':url.strip(),
                'image_count':len([i for i in list(article.images) if 'data:' not in i and 'logo' not in i and 'wpn' not in i and 'author' not in i])}
        article_data.append(article_data_add)
        article_content.append(article_content_add)
        article_image_count.append(article_image_count_add)
        counter+=1
        #print(counter)
        if counter%100==0:
            print(counter,'/',len(url_list),'complete.')
        time.sleep(0.1)
    return article_data,article_content,article_image_count



# +
#test runs
#article_data,article_content,article_image_count=get_article_details(abc_top[0:5],1)
#article_data,article_content,article_image_count=get_article_details(abc_regular[0:5],0)
#article_data,article_content,article_image_count=get_article_details(wsj_top[0:5],1) #sometimes content cant be gathered
#article_data,article_content,article_image_count=get_article_details(wsj_regular[0:5],0)
#article_data,article_content,article_image_count=get_article_details(newsweek_top[0:5],1)
#article_data,article_content,article_image_count=get_article_details(newsweek_regular[0:5],0)
#article_data,article_content,article_image_count=get_article_details(reuters_top[0:5],1)
#article_data,article_content,article_image_count=get_article_details(reuters_regular[0:5],0)
#article_data,article_content,article_image_count=get_article_details(irish_top[0:20],1)
#article_data,article_content,article_image_count=get_article_details(irish_regular[0:20],0)
#article_data,article_content,article_image_count=get_article_details(nytimes_top[0:5],1)
#article_data,article_content,article_image_count=get_article_details(nytimes_regular[0:5],0)
# -

#Full runs
article_data_irish_regular,article_content_irish_regular,article_image_count_irish_regular=get_article_details(irish_regular,0)

import pickle
with open('article_data_irish_regular.pickle', 'wb') as handle:
    pickle.dump(article_data_irish_regular, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('article_content_irish_regular.pickle', 'wb') as handle:
    pickle.dump(article_content_irish_regular, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('article_image_count_irish_regular.pickle', 'wb') as handle:
    pickle.dump(article_image_count_irish_regular, handle, protocol=pickle.HIGHEST_PROTOCOL)



StarcoderdataJupyterScriptsDedupFiltered