Skip to content

Commit

Permalink
Merge pull request #121 from keboola/kacurez-stop-paging-on-empty-res…
Browse files Browse the repository at this point in the history 
…ponse-ST-2131

stop paging on empty response feature
  • Loading branch information
@kacurez
kacurez authored Aug 22, 2024
2 parents 8fa04ab + ab5bd2a commit e974cf0
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 22 deletions.
39 changes: 24 additions & 15 deletions src/keboola/facebook/api/request.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,20 +119,25 @@

(defn get-next-page-url
"return url to the next page from @response param"
[response time-base-pagination?]
(let [next-url (get-in response [:paging :next])]
(if (or (not time-base-pagination?) (and next-url
(not (clojure.string/includes? next-url "since="))
(not (clojure.string/includes? next-url "until="))))
[response time-base-pagination? stop-on-empty-response?]
(let [next-url (get-in response [:paging :next])
time-base-pagination-valid (or (not time-base-pagination?)
(and next-url
(not (clojure.string/includes? next-url "since="))
(not (clojure.string/includes? next-url "until="))))
stop-on-empty-response-valid (or (not stop-on-empty-response?)
(not (-> response :data empty?)))]
(if (not stop-on-empty-response-valid) (log-strings "Found empty data response, stopping pagintaion."))
(if (and time-base-pagination-valid stop-on-empty-response-valid)
next-url)))

(defn get-next-page-data
"if response contains next page url then call it and wait for new repsonse
result: vector with new nested-object like structure
"
[response params ex-account-id top-node time-base-pagination?]
[response params ex-account-id top-node time-base-pagination? stop-on-empty-response?]
;(println "next url" (get-next-page-url response) (:paging response))
(if-let [next-page-url (get-next-page-url response time-base-pagination?)] ; process next api page if exists
(if-let [next-page-url (get-next-page-url response time-base-pagination? stop-on-empty-response?)] ; process next api page if exists
(let [new-response (:body (make-get-request next-page-url))]
(cond (contains? new-response :data)
[{:parent-id (:parent-id params)
Expand All @@ -152,13 +157,13 @@
(defn page-and-collect
"collect data from response and make another paging requests if needed.
Returns lazy sequence of flattened data resulting from processing the whole query"
[{:keys [ex-account-id parent-id fb-graph-node table-name path body-data response] :as init-params} time-base-pagination?]
[{:keys [ex-account-id parent-id fb-graph-node table-name path body-data response] :as init-params} time-base-pagination? stop-on-empty-response?]
((fn step [params this-object-data rest-objects top-node]
(if (and (empty? rest-objects) (empty? this-object-data))
nil
(let [new-rows (mapcat #(extract-values % (dissoc params :body-data :response) ex-account-id) this-object-data)

next-page-data (get-next-page-data (:response params) params ex-account-id top-node time-base-pagination?)
next-page-data (get-next-page-data (:response params) params ex-account-id top-node time-base-pagination? stop-on-empty-response?)
nested-objects (concat (parser/get-nested-objects this-object-data params) next-page-data)
all-objects (concat nested-objects rest-objects)
next-object (first all-objects)
Expand All @@ -181,11 +186,13 @@
query-params (assoc whole-query :access_token access-token :fields preparsed-fields :since preparsed-since :until preparsed-until)
full-url (make-url path version)
time-base-pagination? (:time-based-pagination whole-query)
stop-on-empty-response? (:stop-on-empty-response whole-query)
response (make-get-request full-url query-params)
response-body (:body response)
sanitized-path (keyword (string/replace path #"/" "_"))]

(log-strings "calling" full-url "with" preparsed-fields ids preparsed-since preparsed-until)
(if stop-on-empty-response? (log-strings "Stop on empty response is enabled."))
(if (some? ids)
(mapcat
#(page-and-collect
Expand All @@ -195,7 +202,7 @@
:table-name "page"
:path path
:body-data [(if (not-empty path) {sanitized-path (second %)} (second %))]
:response response-body} time-base-pagination?)
:response response-body} time-base-pagination? stop-on-empty-response?)
response-body)
;else - no ids response
(page-and-collect
Expand All @@ -205,7 +212,7 @@
:table-name "page"
:path path
:body-data [(if (not-empty path) {sanitized-path response-body} response-body)]
:response (if (not-empty path) {sanitized-path response-body} response-body)} time-base-pagination?))))
:response (if (not-empty path) {sanitized-path response-body} response-body)} time-base-pagination? stop-on-empty-response?))))

(defn- job-finished? [poll-response]
(let [status (-> poll-response :body :async_status)
Expand All @@ -225,27 +232,29 @@
(log-strings "Started polling for insights job report:" report_run_id)
(with-exp-backoff finished?-fn)))

(defn- collect-async-insights-result [access-token ad-account-id report-run-id version]
(defn- collect-async-insights-result [access-token ad-account-id report-run-id version stop-on-empty-response?]
(let [url (str (make-url report-run-id version) "/insights/?access_token=" access-token)
response (make-get-request url)
response-body (:body response)]
;(println "response-body" response-body)
(if stop-on-empty-response? (log-strings "Stop on empty response is enabled."))
(page-and-collect
{:ex-account-id ad-account-id
:parent-id ad-account-id
:fb-graph-node "page"
:table-name "page"
:path "insights"
:body-data [{"insights" response-body}]
:response nil} false)))
:response nil} false stop-on-empty-response?)))

(defn async-insights-request [access-token ad-account-id parameters version]
(defn async-insights-request [access-token ad-account-id parameters version query]
(let [url (make-url (str ad-account-id "/insights") version)
stop-on-empty-response? (:stop-on-empty-response query)
url-params (str parameters "&access_token=" access-token)
whole-url (str url "?" url-params)
report-run-id (-> (client/POST whole-url :as :json) :body :report_run_id)]
(poll-async-request report-run-id access-token version)
(collect-async-insights-result access-token ad-account-id report-run-id version)))
(collect-async-insights-result access-token ad-account-id report-run-id version stop-on-empty-response?)))

(defn- collect-result [response api-fn]
(lazy-seq
Expand Down
2 changes: 1 addition & 1 deletion src/keboola/facebook/extractor/query.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@
(defn run-async-insights-query [token out-dir name query version]
(let [ids-str (:ids query)
parameters (:parameters query)
run-query (fn [id] (request/async-insights-request token id parameters version))
run-query (fn [id] (request/async-insights-request token id parameters version query))
ids-seq (s/split ids-str #",")
all-merged-queries-rows (mapcat #(run-query %) ids-seq)
all-rows (apply concat all-merged-queries-rows)]
Expand Down
18 changes: 12 additions & 6 deletions test/keboola/facebook/api/request_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,17 +113,23 @@
(deftest test-get-next-page-url
(let [response {:paging {:previous "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&since=1672915646&until=1675248446&metric=total_interactions&metric_type=total_value&period=day"
:next "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&since=1677581248&until=1679914048&metric=total_interactions&metric_type=total_value&period=day"}}]
(is (nil? (sut/get-next-page-url response true)))
(is (= (sut/get-next-page-url response false) "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&since=1677581248&until=1679914048&metric=total_interactions&metric_type=total_value&period=day")))
(is (nil? (sut/get-next-page-url response true false)))
(is (= (sut/get-next-page-url response false false) "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&since=1677581248&until=1679914048&metric=total_interactions&metric_type=total_value&period=day")))

(let [response {:paging {:previous "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&since=1672915646&until=1675248446&metric=total_interactions&metric_type=total_value&period=day"}}]
(is (nil? (sut/get-next-page-url response true)))
(is (nil? (sut/get-next-page-url response false))))
(is (nil? (sut/get-next-page-url response true false)))
(is (nil? (sut/get-next-page-url response false false))))

(let [response {:paging {:next "someurl"} :data []}]
(is (nil? (sut/get-next-page-url response false true))))

(let [response {:paging {:next "someurl"} :data [{:some "data"}]}]
(is (= "someurl" (sut/get-next-page-url response false true))))

(let [response {:paging {:previous "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&metric=total_interactions&metric_type=total_value&period=day"
:next "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&metric=total_interactions&metric_type=total_value&period=day"}}]
(is (= (sut/get-next-page-url response false) "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&metric=total_interactions&metric_type=total_value&period=day")))
(is (= (sut/get-next-page-url response false false) "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&metric=total_interactions&metric_type=total_value&period=day")))

(let [response {:paging {:previous "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&metric=total_interactions&metric_type=total_value&period=day"
:next "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&metric=reach&metric_type=total_value&period=day"}}]
(is (= (sut/get-next-page-url response false) "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&metric=reach&metric_type=total_value&period=day"))))
(is (= (sut/get-next-page-url response false false) "https://graph.facebook.com/v16.0/17841401480255572/insights?access_token=XXX&pretty=0&metric=reach&metric_type=total_value&period=day"))))

0 comments on commit e974cf0

Please sign in to comment.