added function to sso login, not tested, further development required

src/main/java/elh/eus/MSM/FeedReader.java

@@ -367,6 +367,7 @@ private void getRssFeed (Feed f, String store){
 		try{
 			InputStream stream;
 			String url = f.getFeedURL().trim();
+
 			if (url.startsWith("file://")){
 				stream = new FileInputStream(url.replaceFirst("file://", ""));
 			}
@@ -424,6 +425,20 @@ private void getRssFeed (Feed f, String store){
 			//System.err.println("FeadReader::getFeed -> analysing entries");
 			link = entry.getLink();		
 
+			int domCount=0;
+			Matcher match = MSMUtils.duplicatedDomainInUrl.matcher(link);
+			int matchStart = 0;
+			while (match.find())
+			{
+				domCount=domCount+1;
+				matchStart= match.start();
+			}
+			// check and correct malformed urls  of type https://x.y.comhttps:/x.y.com/page
+			if (domCount > 1) {
+				link = link.substring(matchStart);
+				System.err.println("FeadReader::getRssFeed ->  WARN url with incorrect domain. Clean url: "+link);
+			}
+
 			if (store.equalsIgnoreCase("db"))
 			{	
 				if (MSMUtils.mentionsInDB(DBconn, link) > 0){
@@ -464,6 +479,7 @@ private void getRssFeed (Feed f, String store){
 				{
 					newEnts++;
 					System.err.println("FeadReader::getRssFeed -> new entry "+date+" vs."+f.getLastFetchDate());
+					System.err.println(link);
 					//com.robbypond version.
 					//final BoilerpipeExtractor extractor = CommonExtractors.ARTICLE_EXTRACTOR;
 					//final HtmlArticleExtractor htmlExtr = HtmlArticleExtractor.INSTANCE;
@@ -472,6 +488,7 @@ private void getRssFeed (Feed f, String store){
 					//normal kohlschuetter extractor call
 					// parse the document into boilerpipe's internal data structure
 					//final InputSource is = HTMLFetcher.fetch(linkSrc).toInputSource();
+
 					final InputSource is = fetchHTML(linkSrc, f.getSrcDomain());
 					TextDocument doc = new BoilerpipeSAXInput(is).getTextDocument();
 					// perform the extraction/classification process on "doc"
@@ -491,7 +508,8 @@ private void getRssFeed (Feed f, String store){
 						}
 						else
 						{
-							parseArticleForKeywords(doc,lang, pubDate, link, f.getSrcId(), store);
+							processFullArticle(doc,lang, pubDate, link, f.getSrcId(), store);
+							//parseArticleForKeywords(doc,lang, pubDate, link, f.getSrcId(), store);
 						}
 					}
 				}

src/main/java/elh/eus/MSM/MSMUtils.java

@@ -29,6 +29,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.security.KeyManagementException;
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
@@ -45,16 +46,21 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.regex.Pattern;
 
 import javax.naming.NamingException;
 
+import org.apache.http.Header;
+import org.apache.http.auth.AuthenticationException;
+import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.TrustAllStrategy;
+import org.apache.http.impl.auth.BasicScheme;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.ssl.SSLContextBuilder;
@@ -75,6 +81,9 @@ public final class MSMUtils {
 					new SimpleDateFormat("yyyy-MM-dd HH:mm"),
 					new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z")));
 
+	public static final Pattern duplicatedDomainInUrl = Pattern.compile("http[s]?://");
+
+
 	/**
 	* Check input file integrity.
 	* @param name
@@ -453,4 +462,25 @@ public static CloseableHttpClient httpClient () throws KeyManagementException, N
 
 	}
 
+
+	public static Header SSO_login (String domain, String url, String username, String pass)
+	{	
+		Header result = null;
+		URI authUrl;
+		try {
+			authUrl = new URI(url);
+			HttpUriRequest get = new HttpGet(authUrl);    
+	        result= new BasicScheme(StandardCharsets.UTF_8).authenticate(new UsernamePasswordCredentials(username,pass), get, null);
+
+		} catch (URISyntaxException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (AuthenticationException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		return  result;
+	}
+
+
 }