v2.19 (#400)

* update version * Use try with resources, improve logging (#401) * Use try with resources, improve logging * Use try with resources, improve logging * Replace concatenation with placeholders in logging (#402) * Extract object mapper, optimize user enrichment (#403) * Extract ObjectMapper to JsonHelper * Optimize users enrichment * Add Variant media field (#404) * Add Variant media field * Add assertion * Added TwitterClient.stopFilteredStream() variant with a timeout. (#406) Co-authored-by: Nick <[email protected]> Co-authored-by: takeshitakenji <[email protected]>
redouane59 · Oct 8, 2022 · 23dd6c7 · 23dd6c7
1 parent 14777ba
commit 23dd6c7
Show file tree

Hide file tree

Showing 27 changed files with 251 additions and 153 deletions.
diff --git a/pom.xml b/pom.xml
@@ -194,5 +194,5 @@
 
   <url>https://github.com/Redouane59/twittered</url>
 
-  <version>2.18</version>
+  <version>2.19</version>
 </project>
diff --git a/src/main/java/io/github/redouane59/twitter/ITwitterClientV2.java b/src/main/java/io/github/redouane59/twitter/ITwitterClientV2.java
@@ -23,6 +23,7 @@
 import io.github.redouane59.twitter.dto.user.UserList;
 import java.util.List;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 
 public interface ITwitterClientV2 {
@@ -172,10 +173,20 @@ public interface ITwitterClientV2 {
    */
   Future<Response> startFilteredStream(IAPIEventListener listener, int backfillMinutes);
 
+  /**
+   * Stops the filtered stream with the result of the startFilteredStream. It'll wait a maximum of timeout
+   * before giving up and returning false.  If timeout isn't hit, it'll close the socket opened.
+   *
+   * @param responseFuture Future<Response> given by startFilteredStream
+   * @param timeout long How long to wait
+   * @param unit TimeUnit Units for timeout
+   */
+  boolean stopFilteredStream(Future<Response> response, long timeout, TimeUnit unit);
+
   /**
    * Stops the filtered stream with the result of the startFilteredStream. It'll close the socket opened.
    *
-   * @param response Future<Response> given by startFilteredStream
+   * @param responseFuture Future<Response> given by startFilteredStream
    */
   boolean stopFilteredStream(Future<Response> response);
 
@@ -641,4 +652,4 @@ public interface ITwitterClientV2 {
   boolean deleteTweet(String tweetId);
 
 
-}
+}
diff --git a/src/main/java/io/github/redouane59/twitter/TwitterClient.java b/src/main/java/io/github/redouane59/twitter/TwitterClient.java
diff --git a/src/main/java/io/github/redouane59/twitter/dto/tweet/TweetListDeserializer.java b/src/main/java/io/github/redouane59/twitter/dto/tweet/TweetListDeserializer.java
@@ -4,14 +4,17 @@
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import io.github.redouane59.twitter.TwitterClient;
 import io.github.redouane59.twitter.dto.tweet.TweetList.TweetMeta;
 import io.github.redouane59.twitter.dto.tweet.TweetV2.Includes;
 import io.github.redouane59.twitter.dto.tweet.TweetV2.TweetData;
 import io.github.redouane59.twitter.dto.user.UserV2.UserData;
+import io.github.redouane59.twitter.helpers.JsonHelper;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 public class TweetListDeserializer extends StdDeserializer<TweetList> {
 
@@ -24,20 +27,22 @@ public TweetList deserialize(final JsonParser jsonParser, final DeserializationC
     TweetList result = TweetList.builder().build();
     JsonNode  node   = jsonParser.getCodec().readTree(jsonParser);
     if (node.has("meta")) {
-      result.setMeta(TwitterClient.OBJECT_MAPPER.readValue(node.get("meta").toString(), TweetMeta.class));
+      result.setMeta(JsonHelper.fromJson(node.get("meta"), TweetMeta.class));
     }
     if (node.has("data")) {
-      List<TweetData>
-          list =
-          TwitterClient.OBJECT_MAPPER.readValue(node.get("data").toString(),
-                                                TwitterClient.OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, TweetData.class));
+      List<TweetData> list =
+          JsonHelper.fromJson(node.get("data"), JsonHelper.OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, TweetData.class));
       if (node.has("includes")) {
-        result.setIncludes(TwitterClient.OBJECT_MAPPER.readValue(node.get("includes").toString(), Includes.class));
+        Includes includes = JsonHelper.fromJson(node.get("includes"), Includes.class);
+        result.setIncludes(includes);
+
+        Map<String, UserData> users = includes.getUsers()
+                                              .stream()
+                                              .collect(Collectors.toMap(UserData::getId, Function.identity()));
+
         // in order to enrich the TweetData object (from data field) adding the User object (instead of just the author_id)
         for (TweetData tweetData : list) {
-          Optional<UserData> matchingUser = result.getIncludes().getUsers().stream().filter(p -> p.getId().equals(tweetData.getAuthorId())).
-                                                  findFirst();
-          matchingUser.ifPresent(tweetData::setUser);
+          Optional.ofNullable(users.get(tweetData.getAuthorId())).ifPresent(tweetData::setUser);
         }
       }
       result.setData(list);

diff --git a/src/main/java/io/github/redouane59/twitter/dto/tweet/TweetV1Deserializer.java b/src/main/java/io/github/redouane59/twitter/dto/tweet/TweetV1Deserializer.java
@@ -4,7 +4,7 @@
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import io.github.redouane59.twitter.TwitterClient;
+import io.github.redouane59.twitter.helpers.JsonHelper;
 import java.io.IOException;
 
 
@@ -21,6 +21,6 @@ public TweetV1Deserializer(Class<?> vc) {
   @Override
   public TweetV1 deserialize(JsonParser jp, DeserializationContext deserializationContext) throws IOException {
     JsonNode node = jp.getCodec().readTree(jp);
-    return TwitterClient.OBJECT_MAPPER.readValue(node.get("tweet").toString(), TweetV1.class);
+    return JsonHelper.fromJson(node.get("tweet"), TweetV1.class);
   }
 }
diff --git a/src/main/java/io/github/redouane59/twitter/dto/tweet/TweetV2.java b/src/main/java/io/github/redouane59/twitter/dto/tweet/TweetV2.java
@@ -496,6 +496,8 @@ public static class MediaEntityV2 implements MediaEntity {
     private String                previewImageUrl;
     @JsonProperty("public_metrics")
     private MediaPublicMetricsDTO publicMetrics;
+    @JsonProperty("variants")
+    private List<Variant>         variants;
     @JsonProperty("alt_text")
     private String                altText;
 
@@ -565,4 +567,18 @@ public static class MediaPublicMetricsDTO {
     @JsonProperty("view_count")
     private int viewCount;
   }
+
+  @Getter
+  @Setter
+  public static class Variant {
+
+    @JsonProperty("bit_rate")
+    private int bitRate;
+
+    @JsonProperty("content_type")
+    private String contentType;
+
+    @JsonProperty("url")
+    private String url;
+  }
 }
diff --git a/src/main/java/io/github/redouane59/twitter/helpers/AbstractRequestHelper.java b/src/main/java/io/github/redouane59/twitter/helpers/AbstractRequestHelper.java
@@ -1,14 +1,13 @@
 package io.github.redouane59.twitter.helpers;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.github.scribejava.apis.TwitterApi;
 import com.github.scribejava.core.builder.ServiceBuilder;
 import com.github.scribejava.core.model.OAuthRequest;
 import com.github.scribejava.core.model.Response;
 import com.github.scribejava.core.model.Verb;
 import com.github.scribejava.core.oauth.OAuth10aService;
-import io.github.redouane59.twitter.TwitterClient;
 import io.github.redouane59.twitter.signature.TwitterCredentials;
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.time.format.DateTimeFormatter;
 import java.util.Map;
@@ -42,15 +41,7 @@ protected AbstractRequestHelper(TwitterCredentials twitterCredentials, OAuth10aS
   }
 
   public static void logApiError(String method, String url, String stringResponse, int code) {
-    LOGGER.error("(" + method + ") Error calling " + url + " " + stringResponse + " - " + code);
-  }
-
-  protected <T> T convert(String json, Class<? extends T> targetClass) throws JsonProcessingException {
-    if (targetClass.isInstance(json)) {
-      return (T) json;
-    } else {
-      return TwitterClient.OBJECT_MAPPER.readValue(json, targetClass);
-    }
+    LOGGER.error("({}) Error calling {} {} - {}", method, url, stringResponse, code);
   }
 
   protected abstract void signRequest(OAuthRequest request);
@@ -88,29 +79,33 @@ public <T> Optional<T> makeRequest(OAuthRequest request, boolean signRequired, C
     if (signRequired) {
       signRequest(request);
     }
-    Response response       = getService().execute(request);
-    String   stringResponse = response.getBody();
-    if (response.getCode() == 429) {
-      if (!automaticRetry) {
-        throw new LimitExceededException();
-      }
-      int    retryAfter    = DEFAULT_RETRY_AFTER_SEC;
-      String retryAfterStr = response.getHeader("Retry-After");
-      if (retryAfterStr != null) {
-        try {
-          retryAfter = Integer.parseInt(retryAfterStr);
-        } catch (NumberFormatException e) {
-          LOGGER.error("Using default retry after because header format is invalid: " + retryAfterStr, e);
+    try (Response response = getService().execute(request)) {
+      String stringResponse = response.getBody();
+      if (response.getCode() == 429) {
+        if (!automaticRetry) {
+          throw new LimitExceededException();
+        }
+        int    retryAfter    = DEFAULT_RETRY_AFTER_SEC;
+        String retryAfterStr = response.getHeader("Retry-After");
+        if (retryAfterStr != null) {
+          try {
+            retryAfter = Integer.parseInt(retryAfterStr);
+          } catch (NumberFormatException e) {
+            LOGGER.error("Using default retry after because header format is invalid: {}", retryAfterStr, e);
+          }
         }
+        LOGGER.info("Rate limit exceeded, new retry in {} at {}", ConverterHelper.getSecondsAsText(retryAfter), ConverterHelper.minutesBeforeNow(
+            -retryAfter / 60).format(DateTimeFormatter.ofPattern("HH:mm")));
+        Thread.sleep(1000L * retryAfter);
+        return makeRequest(request, false, classType); // We have already signed if it was requested
+      } else if (response.getCode() < 200 || response.getCode() > 299) {
+        logApiError(request.getVerb().name(), request.getUrl(), stringResponse, response.getCode());
       }
-      LOGGER.info("Rate limit exceeded, new retry in " + ConverterHelper.getSecondsAsText(retryAfter) + " at " + ConverterHelper.minutesBeforeNow(
-          -retryAfter / 60).format(DateTimeFormatter.ofPattern("HH:mm")));
-      Thread.sleep(1000L * retryAfter);
-      return makeRequest(request, false, classType); // We have already signed if it was requested
-    } else if (response.getCode() < 200 || response.getCode() > 299) {
-      logApiError(request.getVerb().name(), request.getUrl(), stringResponse, response.getCode());
+      result = JsonHelper.fromJson(stringResponse, classType);
+    } catch (IOException ex) {
+      LOGGER.error("Error occupied on executing request", ex);
     }
-    result = convert(stringResponse, classType);
+
     return Optional.ofNullable(result);
   }
 

diff --git a/src/main/java/io/github/redouane59/twitter/helpers/JsonHelper.java b/src/main/java/io/github/redouane59/twitter/helpers/JsonHelper.java
@@ -0,0 +1,51 @@
+package io.github.redouane59.twitter.helpers;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.TreeNode;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class JsonHelper {
+
+  public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
+      .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+      .setSerializationInclusion(JsonInclude.Include.NON_NULL)
+      .findAndRegisterModules();
+
+  public static String toJson(Object value) throws JsonProcessingException {
+    return OBJECT_MAPPER.writeValueAsString(value);
+  }
+
+  public static <T> T fromJson(String value, Class<T> clazz) throws JsonProcessingException {
+    return OBJECT_MAPPER.readValue(value, clazz);
+  }
+
+  public static <T> T fromJson(TreeNode node, Class<T> clazz) throws JsonProcessingException {
+    return OBJECT_MAPPER.treeToValue(node, clazz);
+  }
+
+  public static <T> T fromJson(TreeNode node, JavaType javaType) throws JsonProcessingException {
+    return OBJECT_MAPPER.treeToValue(node, javaType);
+  }
+
+  public static <T> T fromJson(String value, JavaType javaType) throws JsonProcessingException {
+    return OBJECT_MAPPER.readValue(value, javaType);
+  }
+
+  /**
+   * Check if the string supplied is valid json
+   */
+  public static boolean isValidJSON(String json) {
+    try {
+      OBJECT_MAPPER.readTree(json);
+      return true;
+    } catch (IOException e) {
+      return false;
+    }
+  }
+}
diff --git a/src/main/java/io/github/redouane59/twitter/helpers/TweetStreamConsumer.java b/src/main/java/io/github/redouane59/twitter/helpers/TweetStreamConsumer.java
@@ -3,7 +3,6 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.github.scribejava.core.model.Response;
 import io.github.redouane59.twitter.IAPIEventListener;
-import io.github.redouane59.twitter.TwitterClient;
 import io.github.redouane59.twitter.dto.tweet.TweetV2;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -37,7 +36,7 @@ public boolean consumeBuffer(String data) {
 
     // Check if the buffer is empty and we receive a valid json data
     if (buffer.toString().isEmpty() && (!data.trim().startsWith("{"))) {
-      LOGGER.warn("Invalid JSON Start Character. Ignoring : " + data);
+      LOGGER.warn("Invalid JSON Start Character. Ignoring : {}", data);
       return false;
     }
     buffer.append(data);
@@ -100,7 +99,7 @@ private <T> boolean handleData(IAPIEventListener listener, final Response respon
     if (response.getCode() == 200) {
       if (clazz == TweetV2.class) {
         try {
-          listener.onTweetStreamed((TweetV2) TwitterClient.OBJECT_MAPPER.readValue(line, clazz));
+          listener.onTweetStreamed((TweetV2) JsonHelper.OBJECT_MAPPER.readValue(line, clazz));
         } catch (JsonProcessingException e) {
           listener.onUnknownDataStreamed(line);
         }
@@ -114,19 +113,6 @@ private <T> boolean handleData(IAPIEventListener listener, final Response respon
     }
   }
 
-
-  /**
-   * Check if the string supplied is valid
-   */
-  private boolean isValidJSON(String json) {
-    try {
-      TwitterClient.OBJECT_MAPPER.readTree(json);
-      return true;
-    } catch (IOException e) {
-      return false;
-    }
-  }
-
   /**
    * Returns an array of string containing 0 to n tweets
    */
@@ -139,8 +125,8 @@ public String[] getJsonTweets() {
 
     // Check if the last result is complete...
     String  lastJSON = result.get(result.size() - 1);
-    boolean complete = isValidJSON(lastJSON);
-    // Reinit the StringBuilder...
+    boolean complete = JsonHelper.isValidJSON(lastJSON);
+    // Re init the StringBuilder...
     buffer = new StringBuilder();
     // If not complete, reconsume the last buffer
     if (!complete) {

diff --git a/src/test/java/io/github/redouane59/twitter/nrt/ITwitterClientV2Test.java b/src/test/java/io/github/redouane59/twitter/nrt/ITwitterClientV2Test.java
@@ -27,6 +27,7 @@
 import io.github.redouane59.twitter.dto.user.User;
 import io.github.redouane59.twitter.dto.user.UserList;
 import io.github.redouane59.twitter.helpers.ConverterHelper;
+import io.github.redouane59.twitter.helpers.JsonHelper;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -64,7 +65,7 @@ public void getUserByUserName() {
   public void getAndSerializeUser() throws IOException {
     String userName = "RedouaneBali";
     User   result   = twitterClient.getUserFromUserName(userName);
-    assertNotNull(TwitterClient.OBJECT_MAPPER.writeValueAsString(result));
+    assertNotNull(JsonHelper.toJson(result));
   }
 
   @Test

diff --git a/src/test/java/io/github/redouane59/twitter/unit/BearerTokenSereializerTest.java b/src/test/java/io/github/redouane59/twitter/unit/BearerTokenSereializerTest.java
@@ -3,8 +3,8 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-import io.github.redouane59.twitter.TwitterClient;
 import io.github.redouane59.twitter.dto.others.BearerToken;
+import io.github.redouane59.twitter.helpers.JsonHelper;
 import io.github.redouane59.twitter.signature.Scope;
 import java.io.File;
 import java.io.IOException;
@@ -14,7 +14,7 @@
 public class BearerTokenSereializerTest {
 
   private File        bearerFile  = new File(getClass().getClassLoader().getResource("tests/bearer_token_example.json").getFile());
-  private BearerToken bearerToken = TwitterClient.OBJECT_MAPPER.readValue(bearerFile, BearerToken.class);
+  private BearerToken bearerToken = JsonHelper.OBJECT_MAPPER.readValue(bearerFile, BearerToken.class);
 
   public BearerTokenSereializerTest() throws IOException {
   }