Refresh latest

pom.xml

@@ -16,7 +16,7 @@
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-compiler-plugin</artifactId>
-				<version>3.8.0</version>
+				<version>3.8.1</version>
 				<configuration>
 					<source>1.8</source>
 					<target>1.8</target>
@@ -25,7 +25,7 @@
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-shade-plugin</artifactId>
-				<version>3.2.0</version>
+				<version>3.2.2</version>
 				<configuration>
 					<filters>
 						<filter>
@@ -66,7 +66,7 @@
 		<dependency>
 			<groupId>com.squareup.okhttp3</groupId>
 			<artifactId>okhttp</artifactId>
-			<version>3.12.0</version>
+			<version>4.4.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.projectlombok</groupId>
@@ -78,13 +78,13 @@
 		<dependency>
 			<groupId>commons-beanutils</groupId>
 			<artifactId>commons-beanutils</artifactId>
-			<version>1.9.3</version>
+			<version>1.9.4</version>
 		</dependency>
 		<!-- https://mvnrepository.com/artifact/org.eclipse.persistence/org.eclipse.persistence.moxy -->
 		<dependency>
 			<groupId>org.eclipse.persistence</groupId>
 			<artifactId>org.eclipse.persistence.moxy</artifactId>
-			<version>2.5.0</version>
+			<version>2.7.6</version>
 		</dependency>
 		<!-- https://mvnrepository.com/artifact/org.hibernate.javax.persistence/hibernate-jpa-2.1-api -->
 		<dependency>
@@ -96,44 +96,44 @@
 		<dependency>
 			<groupId>commons-io</groupId>
 			<artifactId>commons-io</artifactId>
-			<version>2.6</version>
+			<version>2.7</version>
 		</dependency>
 		<!-- https://mvnrepository.com/artifact/com.github.igor-suhorukov/dom-transformation -->
 		<dependency>
 			<groupId>com.github.igor-suhorukov</groupId>
 			<artifactId>dom-transformation</artifactId>
-			<version>1.1</version>
+			<version>1.2</version>
 		</dependency>
 		<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
 		<dependency>
 			<groupId>com.fasterxml.jackson.core</groupId>
 			<artifactId>jackson-databind</artifactId>
-			<version>2.9.7</version>
+			<version>2.10.0.pr1</version>
 		</dependency>
 		<!-- https://mvnrepository.com/artifact/com.squareup.okhttp3/logging-interceptor -->
 		<dependency>
 			<groupId>com.squareup.okhttp3</groupId>
 			<artifactId>logging-interceptor</artifactId>
-			<version>3.11.0</version>
+			<version>4.4.0</version>
 		</dependency>
 		<!-- https://mvnrepository.com/artifact/org.assertj/assertj-core -->
 		<dependency>
 			<groupId>org.assertj</groupId>
 			<artifactId>assertj-core</artifactId>
-			<version>3.11.1</version>
+			<version>3.15.0</version>
 			<scope>test</scope>
 		</dependency>
 		<!-- https://mvnrepository.com/artifact/javax.xml.bind/jaxb-api -->
 		<dependency>
 			<groupId>javax.xml.bind</groupId>
 			<artifactId>jaxb-api</artifactId>
-			<version>2.1</version>
+			<version>2.3.1</version>
 		</dependency>
 		<!-- https://mvnrepository.com/artifact/junit/junit -->
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>
-			<version>4.12</version>
+			<version>4.13.1</version>
 			<scope>test</scope>
 		</dependency>
 	</dependencies>

src/main/java/me/postaddict/instagram/scraper/AuthenticatedInsta.java

@@ -25,4 +25,6 @@ public interface AuthenticatedInsta extends AnonymousInsta {
     PageObject<Account> getFollowers(long userId, int pageCount) throws IOException;
 
     ActivityFeed getActivityFeed() throws IOException;
+
+    Long getLoginUserId();
 }

src/main/java/me/postaddict/instagram/scraper/ErrorType.java

@@ -0,0 +1,22 @@
+package me.postaddict.instagram.scraper;
+
+public enum ErrorType {
+
+    CHECKPOINT_REQUIRED,
+
+    TWO_FACTOR_REQUIRED,
+
+    UNAUTHORIZED,
+
+    TEMPORARY_ACTION_BLOCKED,
+
+    ACTION_BLOCKED,
+
+    FOLLOWING_THE_MAX_LIMIT_OF_ACCOUNTS,
+
+    RATE_LIMITED,
+
+    INSTAGRAM_SERVER_ERROR,
+
+    UNKNOWN_ERROR
+}

src/main/java/me/postaddict/instagram/scraper/Instagram.java

@@ -30,7 +30,9 @@
 import me.postaddict.instagram.scraper.request.parameters.MediaCode;
 import me.postaddict.instagram.scraper.request.parameters.TagName;
 import me.postaddict.instagram.scraper.request.parameters.UserParameter;
+import okhttp3.Cookie;
 import okhttp3.FormBody;
+import okhttp3.HttpUrl;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import okhttp3.RequestBody;
@@ -53,8 +55,10 @@ public Instagram(OkHttpClient httpClient) {
 
     protected Request withCsrfToken(Request request) {
     	return request.newBuilder()
-              .addHeader("X-CSRFToken", csrf_token)
+              .addHeader("X-CSRFToken", getCSRFToken())
               .addHeader("X-Instagram-AJAX", (rollout_hash.isEmpty() ? "1" : rollout_hash))
+              .addHeader("X-Requested-With", "XMLHttpRequest")
+              .addHeader("X-IG-App-ID", "936619743392459")
               .build();
     }
 
@@ -76,6 +80,15 @@ private void getCSRFToken(ResponseBody body) throws IOException {
     	this.csrf_token=getToken("\"csrf_token\":\"",32,body.byteStream());
     }
 
+    private String getCSRFToken() {
+        for (Cookie cookie : this.httpClient.cookieJar().loadForRequest(HttpUrl.parse(Endpoint.BASE_URL))) {
+            if ("csrftoken".equals(cookie.name())) {
+                return cookie.value();
+            }
+        }
+        return csrf_token;
+    }
+
     private void getRolloutHash(ResponseBody body){
     	try {
 			this.rollout_hash=getToken("\"rollout_hash\":\"",12,body.byteStream());
@@ -107,18 +120,20 @@ public void login(String username, String password) throws IOException {
         RequestBody formBody = new FormBody.Builder()
                 .add("username", username)
                 .add("password", password)
+                .add("queryParams", "{}")
+                .add("optIntoOneTap", "true")
                 .build();
 
         Request request = new Request.Builder()
                 .url(Endpoint.LOGIN_URL)
-                .header(Endpoint.REFERER, Endpoint.BASE_URL + "/")
+                .header(Endpoint.REFERER, Endpoint.BASE_URL + "/accounts/login/")
                 .post(formBody)
                 .build();
 
         Response response = executeHttpRequest(withCsrfToken(request));
         try(InputStream jsonStream = response.body().byteStream()) {
             if(!mapper.isAuthenticated(jsonStream)){
-                throw new InstagramAuthException("Credentials rejected by instagram");
+                throw new InstagramAuthException("Credentials rejected by instagram", ErrorType.UNAUTHORIZED);
             }
         }
     }
@@ -130,7 +145,7 @@ public Account getAccountById(long id) throws IOException {
                 .build();
         Response response = executeHttpRequest(withCsrfToken(request));
         try(InputStream jsonStream = response.body().byteStream()) {
-            return getMediaByCode(mapper.getLastMediaShortCode(jsonStream)).getOwner();
+            return getAccountByUsername(getMediaByCode(mapper.getLastMediaShortCode(jsonStream)).getOwner().getUsername());
         }
     }
 
@@ -344,4 +359,14 @@ private void validateTagName(String tag) {
             throw new IllegalArgumentException("Please provide non empty tag name that not starts with #");
         }
     }
+
+    @Override
+    public Long getLoginUserId() {
+        for (Cookie cookie : this.httpClient.cookieJar().loadForRequest(HttpUrl.parse(Endpoint.BASE_URL))) {
+            if ("ds_user_id".equals(cookie.name())) {
+                return Long.parseLong(cookie.value());
+            }
+        }
+        return null;
+    }
 }

src/main/java/me/postaddict/instagram/scraper/InstagramFactory.java

@@ -4,7 +4,7 @@
 import me.postaddict.instagram.scraper.cookie.CookieHashSet;
 import me.postaddict.instagram.scraper.cookie.DefaultCookieJar;
 import me.postaddict.instagram.scraper.interceptor.ErrorInterceptor;
-import me.postaddict.instagram.scraper.interceptor.UserAgentInterceptor;
+import me.postaddict.instagram.scraper.interceptor.FakeBrowserInterceptor;
 import okhttp3.OkHttpClient;
 
 import java.io.IOException;
@@ -16,7 +16,7 @@ public static Instagram getAuthenticatedInstagramClient(String login, String pas
             throws IOException{
 
         OkHttpClient httpClient = new OkHttpClient.Builder()
-                .addInterceptor(new UserAgentInterceptor(userAgent))
+                .addInterceptor(new FakeBrowserInterceptor(userAgent))
                 .addInterceptor(new ErrorInterceptor())
                 .cookieJar(new DefaultCookieJar(new CookieHashSet()))
                 .build();

src/main/java/me/postaddict/instagram/scraper/exception/InstagramAuthException.java

@@ -1,10 +1,17 @@
 package me.postaddict.instagram.scraper.exception;
 
+import me.postaddict.instagram.scraper.ErrorType;
+
 public class InstagramAuthException extends InstagramException {
 
-    public InstagramAuthException(){}
+    public InstagramAuthException() {
+    }
 
     public InstagramAuthException(String message) {
-        super(message);
+        super(message, ErrorType.UNKNOWN_ERROR);
+    }
+
+    public InstagramAuthException(String message, ErrorType errorType) {
+        super(message, errorType);
     }
 }

src/main/java/me/postaddict/instagram/scraper/exception/InstagramException.java

@@ -2,12 +2,26 @@
 
 import java.io.IOException;
 
+import me.postaddict.instagram.scraper.ErrorType;
+
 public class InstagramException extends IOException {
+
+    private ErrorType errorType;
 
     public InstagramException() {
     }
 
-    public InstagramException(String message) {
+    public InstagramException(String message, ErrorType errorType) {
         super(message);
+        this.errorType = errorType;
+    }
+
+    public ErrorType getErrorType() {
+        return errorType;
+    }
+
+    @Override
+    public String getMessage() {
+        return super.getMessage() + " : " + getErrorType();
     }
 }

src/main/java/me/postaddict/instagram/scraper/exception/InstagramNotFoundException.java

@@ -1,11 +1,17 @@
 package me.postaddict.instagram.scraper.exception;
 
+import me.postaddict.instagram.scraper.ErrorType;
+
 public class InstagramNotFoundException extends InstagramException {
 
     public InstagramNotFoundException() {
     }
 
     public InstagramNotFoundException(String message) {
-        super(message);
+        super(message, ErrorType.UNKNOWN_ERROR);
+    }
+
+    public InstagramNotFoundException(String message, ErrorType errorType) {
+        super(message, errorType);
     }
 }

src/main/java/me/postaddict/instagram/scraper/interceptor/ErrorInterceptor.java

@@ -1,5 +1,6 @@
 package me.postaddict.instagram.scraper.interceptor;
 
+import me.postaddict.instagram.scraper.ErrorType;
 import me.postaddict.instagram.scraper.exception.InstagramAuthException;
 import me.postaddict.instagram.scraper.exception.InstagramException;
 import me.postaddict.instagram.scraper.exception.InstagramNotFoundException;
@@ -17,18 +18,43 @@ public Response intercept(Chain chain) throws IOException {
         if (code == 200) {
             return response;
         } else {
+            String bodyString = String.valueOf(response.body().string());
             response.body().close();
-        }
 
-        switch (code) {
-            case 401:
-                throw new InstagramAuthException("Unauthorized");
-            case 403:
-                throw new InstagramAuthException("Access denied");
-            case 404:
-                throw new InstagramNotFoundException("Resource does not exist");
-            default:
-                throw new InstagramException("Response code is not equal 200. Something went wrong. Please report issue.");
+            ErrorType errorType = ErrorType.UNKNOWN_ERROR;
+            switch (code) {
+                case 400:
+                    if (bodyString.contains("Sorry, you're following the max limit of accounts.")) {
+                        errorType = ErrorType.FOLLOWING_THE_MAX_LIMIT_OF_ACCOUNTS;
+                    } else if (bodyString.contains("feedback_required") ||
+                            bodyString.contains("Action Blocked") ||
+                            bodyString.contains("\\u30d6\\u30ed\\u30c3\\u30af\\u3055\\u308c\\u3066\\u3044\\u307e\\u3059")) {
+                        errorType = ErrorType.ACTION_BLOCKED;
+                    } else if (bodyString.contains("checkpoint_required")) {
+                        errorType = ErrorType.CHECKPOINT_REQUIRED;
+                    } else if (bodyString.contains("two_factor_required")) {
+                        errorType = ErrorType.TWO_FACTOR_REQUIRED;
+                    }
+                    throw new InstagramException("Bad Request", errorType);
+                case 401:
+                    throw new InstagramAuthException("Unauthorized", ErrorType.UNAUTHORIZED);
+                case 403:
+                    if (bodyString.contains("Please wait a few minutes before you try again.") ||
+                            bodyString.contains("数分してからもう一度実行してください。")) {
+                        errorType = ErrorType.TEMPORARY_ACTION_BLOCKED;
+                    } else if (bodyString.contains("unauthorized")) {
+                        errorType = ErrorType.UNAUTHORIZED;
+                    }
+                    throw new InstagramAuthException("Access denied", errorType);
+                case 404:
+                    throw new InstagramNotFoundException("Resource does not exist", errorType);
+                case 429:
+                    throw new InstagramException("Rate limited", ErrorType.RATE_LIMITED);
+                case 502:
+                    throw new InstagramException("Bad Gateway", ErrorType.INSTAGRAM_SERVER_ERROR);
+                default:
+                    throw new InstagramException("Response code is " + code + ".", errorType);
+            }
         }
     }
 }

src/main/java/me/postaddict/instagram/scraper/interceptor/UserAgentInterceptor.java → src/main/java/me/postaddict/instagram/scraper/interceptor/FakeBrowserInterceptor.java

@@ -6,11 +6,11 @@
 
 import java.io.IOException;
 
-public class UserAgentInterceptor implements Interceptor {
+public class FakeBrowserInterceptor implements Interceptor {
 
     private final String userAgent;
 
-    public UserAgentInterceptor(String userAgent) {
+    public FakeBrowserInterceptor(String userAgent) {
         this.userAgent = userAgent;
     }
 
@@ -23,6 +23,9 @@ public Response intercept(Chain chain) throws IOException {
         Request originalRequest = chain.request();
         Request newRequest = originalRequest.newBuilder()
                 .header("User-Agent", userAgent)
+                .header("Accept", "*/*")
+                .header("Accept-Language", "ja,en-US;q=0.7,en;q=0.3")
+                .header("DNT", "1")
                 .build();
         return chain.proceed(newRequest);
     }

src/main/java/me/postaddict/instagram/scraper/mapper/ModelMapper.java

@@ -48,7 +48,9 @@ public Media mapMedia(InputStream jsonStream){
         GraphQlResponse<Media> graphQlResponse = mapObject(jsonStream,
                 "me/postaddict/instagram/scraper/model/media-by-url.json");
         Media media = graphQlResponse.getPayload();
-        media.setCommentCount(media.getCommentPreview().getCount());
+        if (media.getCommentPreview() != null) {
+            media.setCommentCount(media.getCommentPreview().getCount());
+        }
         if(media.getCommentPreview()!=null && media.getCommentPreview().getNodes()!=null) {
             media.getCommentPreview().getNodes().forEach(this::updateCommentTime);
         }

src/main/java/me/postaddict/instagram/scraper/model/Account.java

@@ -40,4 +40,5 @@ public class Account {
     @Transient
     private PageObject<Media> media;
     private Date lastUpdated = new Date();
+    private Boolean isBusinessAccount;
 }