@Test public void crawl() throws Exception { setHashTags(); twitterCrawler.crawl(); }
@Override public void crawl() { if (hashTags == null) { loadHashTags(); } if (twitter == null) { configureTwitterAuthentication(); } List<Post> positivePosts = new ArrayList<>(); List<TweetBundle> tweetBundles = getRaw(); for (TweetBundle bundle : tweetBundles) { positivePosts.addAll(rawToPosts(bundle)); } logger.info("Filtered out " + (LOAD_TWEETS_PER_HASTAG * hashTags.length - positivePosts.size()) + " out of " + LOAD_TWEETS_PER_HASTAG * hashTags.length + " tweets"); if (positivePosts.size() >= MAX_TWEETS) { List<Post> postsToSave = positivePosts.subList(0, MAX_TWEETS); logger.info("Saving " + MAX_TWEETS + " tweets to the database"); savePosts(postsToSave); } else { logger.info("Saving " + positivePosts.size() + " tweets to the database"); savePosts(positivePosts); } }
TwitterCrawler extends Crawler<TweetBundle> { @Override public void crawl() { if (hashTags == null) { loadHashTags(); } if (twitter == null) { configureTwitterAuthentication(); } List<Post> positivePosts = new ArrayList<>(); List<TweetBundle> tweetBundles = getRaw(); for (TweetBundle bundle : tweetBundles) { positivePosts.addAll(rawToPosts(bundle)); } logger.info("Filtered out " + (LOAD_TWEETS_PER_HASTAG * hashTags.length - positivePosts.size()) + " out of " + LOAD_TWEETS_PER_HASTAG * hashTags.length + " tweets"); if (positivePosts.size() >= MAX_TWEETS) { List<Post> postsToSave = positivePosts.subList(0, MAX_TWEETS); logger.info("Saving " + MAX_TWEETS + " tweets to the database"); savePosts(postsToSave); } else { logger.info("Saving " + positivePosts.size() + " tweets to the database"); savePosts(positivePosts); } } }
TwitterCrawler extends Crawler<TweetBundle> { @Override public void crawl() { if (hashTags == null) { loadHashTags(); } if (twitter == null) { configureTwitterAuthentication(); } List<Post> positivePosts = new ArrayList<>(); List<TweetBundle> tweetBundles = getRaw(); for (TweetBundle bundle : tweetBundles) { positivePosts.addAll(rawToPosts(bundle)); } logger.info("Filtered out " + (LOAD_TWEETS_PER_HASTAG * hashTags.length - positivePosts.size()) + " out of " + LOAD_TWEETS_PER_HASTAG * hashTags.length + " tweets"); if (positivePosts.size() >= MAX_TWEETS) { List<Post> postsToSave = positivePosts.subList(0, MAX_TWEETS); logger.info("Saving " + MAX_TWEETS + " tweets to the database"); savePosts(postsToSave); } else { logger.info("Saving " + positivePosts.size() + " tweets to the database"); savePosts(positivePosts); } } TwitterCrawler(); }
TwitterCrawler extends Crawler<TweetBundle> { @Override public void crawl() { if (hashTags == null) { loadHashTags(); } if (twitter == null) { configureTwitterAuthentication(); } List<Post> positivePosts = new ArrayList<>(); List<TweetBundle> tweetBundles = getRaw(); for (TweetBundle bundle : tweetBundles) { positivePosts.addAll(rawToPosts(bundle)); } logger.info("Filtered out " + (LOAD_TWEETS_PER_HASTAG * hashTags.length - positivePosts.size()) + " out of " + LOAD_TWEETS_PER_HASTAG * hashTags.length + " tweets"); if (positivePosts.size() >= MAX_TWEETS) { List<Post> postsToSave = positivePosts.subList(0, MAX_TWEETS); logger.info("Saving " + MAX_TWEETS + " tweets to the database"); savePosts(postsToSave); } else { logger.info("Saving " + positivePosts.size() + " tweets to the database"); savePosts(positivePosts); } } TwitterCrawler(); @Override void crawl(); List<Post> rawToPosts(TweetBundle tweets); }
TwitterCrawler extends Crawler<TweetBundle> { @Override public void crawl() { if (hashTags == null) { loadHashTags(); } if (twitter == null) { configureTwitterAuthentication(); } List<Post> positivePosts = new ArrayList<>(); List<TweetBundle> tweetBundles = getRaw(); for (TweetBundle bundle : tweetBundles) { positivePosts.addAll(rawToPosts(bundle)); } logger.info("Filtered out " + (LOAD_TWEETS_PER_HASTAG * hashTags.length - positivePosts.size()) + " out of " + LOAD_TWEETS_PER_HASTAG * hashTags.length + " tweets"); if (positivePosts.size() >= MAX_TWEETS) { List<Post> postsToSave = positivePosts.subList(0, MAX_TWEETS); logger.info("Saving " + MAX_TWEETS + " tweets to the database"); savePosts(postsToSave); } else { logger.info("Saving " + positivePosts.size() + " tweets to the database"); savePosts(positivePosts); } } TwitterCrawler(); @Override void crawl(); List<Post> rawToPosts(TweetBundle tweets);  }