updated user info. bug fixes. Version update

pom.xml

@@ -188,5 +188,5 @@
 		</plugins>
 	</build>
 	<url>https://github.com/Elhuyar/MSM.git</url>
-	<version>1.3.3</version>
+	<version>1.3.4</version>
 </project>

src/main/java/elh/eus/MSM/CLI.java

@@ -524,11 +524,12 @@ public final void twtUserUInfo()
 		String cfg = parsedArguments.getString("config");
 		String store = parsedArguments.getString("store");
 		Integer limit = parsedArguments.getInt("limit");
+		Boolean onlyffCount = parsedArguments.getBoolean("onlyffCount");
 
 		//String params = parsedArguments.getString("params");
 
 		try {
-			TwtUserInfo userInfoClient = new TwtUserInfo(cfg, store,limit);
+			TwtUserInfo userInfoClient = new TwtUserInfo(cfg, store,limit,onlyffCount);
 		} catch (Exception e) {			
 			e.printStackTrace();
 		} 		
@@ -675,6 +676,10 @@ public final void loadTwitterUserInfoParameters()
 		.help("limit the number of users processed in the execution (only for database interaction): default is 500\n"
 				+ "--limit = 0 means no limit is established, and thus the command will atempt to process all sources found in the db (not processed yet).\n"
 				+ "This parameter is important depending on the number of APIs you have available and your usage rate limits.\n");
+		twitterUserParser.addArgument("-o", "--onlyffCount")		
+		.action(Arguments.storeTrue())
+		.help("If this flag is active only follower and friend info will be returned, but no follower and friends lists. "
+				+ "Returning only follower and friends count is much faster because of the higher rate limit of the API.\n");
 	}
 
 	public final void loadUserLocationGeocoderParameters()

src/main/java/elh/eus/MSM/TwtUserInfo.java

@@ -20,7 +20,9 @@ public class TwtUserInfo {
 
 	private Properties params = new Properties();	
 	private String store = "";
-
+	private long wait = 1000*61; 
+	/*1000 milliseconds is one second. Default wait time between requests is 61 seconds, 
+	 * because the more restrictive rate limit of the APIs consulted is 60 seconds */ 	 	
 	public String getStore() {
 		return store;
 	}
@@ -29,15 +31,35 @@ public void setStore(String store) {
 		this.store = store;
 	}
 
+	public long getWait() {
+		return  wait;
+	}
+
+	public void setWait(long wait) {
+		this.wait = wait;
+	}
 
 	/**
 	 * 
 	 * Constructor and main function. Starts a connections and gives the messages to the corresponding handlers.
 	 * 
+	 * 
+	 * The function makes use of twitter's GET users/show API (https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-users-show)
+	 * 
+	 * Rate limits: 
+	 * Requests / 15-min window (user auth) 	900
+	 * Requests / 15-min window (app auth)		900
+	 *  	
+	 * (https://api.twitter.com/1.1/users/show.json)
+	 * 
+	 * However, we also ask for the 
+	 * GET followers/ids 	followers 	15 	15
+	 * GET friends/ids 	friends 	15 	15
+	 * 
 	 * @param config
 	 * @param store
 	 */
-	public TwtUserInfo(String config, String store, Integer limit)
+	public TwtUserInfo(String config, String store, Integer limit, boolean onlyffCounts)
 	{
 		try {
 			params.load(new FileInputStream(new File(config)));
@@ -49,9 +71,15 @@ public TwtUserInfo(String config, String store, Integer limit)
 			System.exit(1);
 		} 
 
+
 		//where should the retrieved messages be stored [db|solr|stout]
 		setStore(store);
 
+		if (onlyffCounts)
+		{
+			setWait(2000);
+		}
+
 		List<Long> usrIds = new ArrayList<Long>();	
 		//Config file settings have preference over DB, if config file parameter is null then try the DB
 		if (!params.getProperty("userIds","none").equalsIgnoreCase("none"))
@@ -112,31 +140,48 @@ public TwtUserInfo(String config, String store, Integer limit)
 				int ff = u.getFollowersCount();
 				int fr = u.getFriendsCount();
 
-				long[] friends = twitter.getFriendsIDs(id, -1).getIDs();
-				long[] followers = twitter.getFollowersIDs(id, -1).getIDs();
-
 				System.out.println("--> "+id+"\t"+ff+"\t"+fr);
 
-				//print format: id1 '\t' id2 --> id2 is follower of id1
-				//print followers
-				for (long i : followers)
+				if (!onlyffCounts)
 				{
-					System.out.println(id+"\t"+i);
-				}
+					long[] friends = twitter.getFriendsIDs(id, -1).getIDs();
+					long[] followers = twitter.getFollowersIDs(id, -1).getIDs();
 
-				//print friends
-				for (long i : friends)
-				{
-					System.out.println(i+"\t"+id);
+					//print format: id1 '\t' id2 --> id2 is follower of id1
+					//print followers
+					for (long i : followers)
+					{
+						System.out.println(id+"\t"+i);
+					}
+
+					//print friends
+					for (long i : friends)
+					{
+						System.out.println(i+"\t"+id);
+					}
 				}
 			}catch (TwitterException twte){
-				System.err.println("MSM::TwtUserInfo - Error when retrieving user info from Twitter id: "+id);
-				//System.out.println("--> "+id+"\t-1\t-1");
-				twte.printStackTrace();					
+				if (twte.exceededRateLimitation())
+				{
+					System.err.println("MSM::TwtUserInfo - Rate limite exceeded with id: "
+							+id+"MSM will wait 15 minutes before the next call");					
+					try {
+						Thread.sleep(1000*60*15);
+					} catch (InterruptedException e) {
+						System.err.println("MSM::TwtUserInfo - Error when waiting after rate limit exceeded.");			
+						e.printStackTrace();
+					}
+				}
+				else
+				{
+					System.err.println("MSM::TwtUserInfo - Error when retrieving user info from Twitter id: "+id);
+					//System.out.println("--> "+id+"\t-1\t-1");
+					twte.printStackTrace();
+				}				
 			}		
 			try
 			{
-				Thread.sleep(1000*61); //1000 milliseconds is one second.					
+				Thread.sleep(getWait()); 				
 			} catch (InterruptedException inte) {
 				System.err.println("MSM::TwtUserInfo - Error when waiting for the next call to Twitter");			
 				inte.printStackTrace();