twitter: implemented filter based group chats

Filter group chats allow for the ability to read the tweets of select
users without actually following the users, and/or track keywords or
hashtags. A filter group chat can have multiple users, keywords, or
hashtags. These users, keywords, or hashtags can span multiple group
chats. This allows for rather robust filter organization.

The underlying structure for the filters is based on linked list, as
using the glib hash tables requires >= glib-2.16 for sanity. Since the
glib requirement of bitlbee is only 2.14, linked list are used in order
to prevent an overly complex implementation.

The idea for this patch was inspired by Artem Savkov's "Twitter search
channels" patch.

In order to use the filter group chats, a group chat must be added to
the twitter account. The channel room name is either follow:username,
track:keyword, and/or track:#hashtag. Multiple elements can be used by
separating each element by a semicolon.

4 files changed, 503 insertions, 7 deletions

diff --git a/protocols/twitter/twitter.c b/protocols/twitter/twitter.c
index d2aafcb4..edc81427 100644
--- a/protocols/twitter/twitter.c
+++ b/protocols/twitter/twitter.c
@@ -30,6 +30,215 @@
 #include "url.h"
 
 GSList *twitter_connections = NULL;
+
+static int twitter_filter_cmp(struct twitter_filter *tf1,
+                              struct twitter_filter *tf2)
+{
+	int i1 = 0;
+	int i2 = 0;
+	int i;
+
+	static const twitter_filter_type_t types[] = {
+		/* Order of the types */
+		TWITTER_FILTER_TYPE_FOLLOW,
+		TWITTER_FILTER_TYPE_TRACK
+	};
+
+	for (i = 0; i < G_N_ELEMENTS(types); i++) {
+		if (types[i] == tf1->type) {
+			i1 = i + 1;
+			break;
+		}
+	}
+
+	for (i = 0; i < G_N_ELEMENTS(types); i++) {
+		if (types[i] == tf2->type) {
+			i2 = i + 1;
+			break;
+		}
+	}
+
+	if (i1 != i2) {
+		/* With different types, return their difference */
+		return i1 - i2;
+	}
+
+	/* With the same type, return the text comparison */
+	return g_strcasecmp(tf1->text, tf2->text);
+}
+
+static gboolean twitter_filter_update(gpointer data, gint fd,
+                                      b_input_condition cond)
+{
+	struct im_connection *ic = data;
+	struct twitter_data *td = ic->proto_data;
+
+	if (td->filters) {
+		twitter_open_filter_stream(ic);
+	} else if (td->filter_stream) {
+		http_close(td->filter_stream);
+		td->filter_stream = NULL;
+	}
+
+	td->filter_update_id = 0;
+	return FALSE;
+}
+
+static struct twitter_filter *twitter_filter_get(struct groupchat *c,
+                                                 twitter_filter_type_t type,
+                                                 const char *text)
+{
+	struct twitter_data *td = c->ic->proto_data;
+	struct twitter_filter *tf = NULL;
+	struct twitter_filter tfc = {type, (char*) text};
+	GSList *l;
+
+	for (l = td->filters; l; l = g_slist_next(l)) {
+		tf = l->data;
+
+		if (twitter_filter_cmp(tf, &tfc) == 0)
+			break;
+
+		tf = NULL;
+	}
+
+	if (!tf) {
+		tf = g_new0(struct twitter_filter, 1);
+		tf->type = type;
+		tf->text = g_strdup(text);
+		td->filters = g_slist_prepend(td->filters, tf);
+	}
+
+	if (!g_slist_find(tf->groupchats, c))
+		tf->groupchats = g_slist_prepend(tf->groupchats, c);
+
+	if (td->filter_update_id > 0)
+		b_event_remove(td->filter_update_id);
+
+	/* Wait for other possible filter changes to avoid request spam */
+	td->filter_update_id = b_timeout_add(TWITTER_FILTER_UPDATE_WAIT,
+					     twitter_filter_update, c->ic);
+	return tf;
+}
+
+static void twitter_filter_free(struct twitter_filter *tf)
+{
+	g_slist_free(tf->groupchats);
+	g_free(tf->text);
+	g_free(tf);
+}
+
+static void twitter_filter_remove(struct groupchat *c)
+{
+	struct twitter_data *td = c->ic->proto_data;
+	struct twitter_filter *tf;
+	GSList *l = td->filters;
+	GSList *p;
+
+	while (l != NULL) {
+		tf = l->data;
+		tf->groupchats = g_slist_remove(tf->groupchats, c);
+
+		p = l;
+		l = g_slist_next(l);
+
+		if (!tf->groupchats) {
+			twitter_filter_free(tf);
+			td->filters = g_slist_delete_link(td->filters, p);
+		}
+	}
+
+	if (td->filter_update_id > 0)
+		b_event_remove(td->filter_update_id);
+
+	/* Wait for other possible filter changes to avoid request spam */
+	td->filter_update_id = b_timeout_add(TWITTER_FILTER_UPDATE_WAIT,
+					     twitter_filter_update, c->ic);}
+
+static void twitter_filter_remove_all(struct im_connection *ic)
+{
+	struct twitter_data *td = ic->proto_data;
+	GSList *chats = NULL;
+	struct twitter_filter *tf;
+	GSList *l = td->filters;
+	GSList *p;
+
+	while (l != NULL) {
+		tf = l->data;
+
+		/* Build up a list of groupchats to be freed */
+		for (p = tf->groupchats; p; p = g_slist_next(p)) {
+			if (!g_slist_find(chats, p->data))
+				chats = g_slist_prepend(chats, p->data);
+		}
+
+		p = l;
+		l = g_slist_next(l);
+		twitter_filter_free(p->data);
+		td->filters = g_slist_delete_link(td->filters, p);
+	}
+
+	l = chats;
+
+	while (l != NULL) {
+		p = l;
+		l = g_slist_next(l);
+
+		/* Freed each remaining groupchat */
+		imcb_chat_free(p->data);
+		chats = g_slist_delete_link(chats, p);
+	}
+
+	if (td->filter_stream) {
+		http_close(td->filter_stream);
+		td->filter_stream = NULL;
+	}
+}
+
+static GSList *twitter_filter_parse(struct groupchat *c, const char *text)
+{
+	char **fs = g_strsplit(text, ";", 0);
+	GSList *ret = NULL;
+	struct twitter_filter *tf;
+	char **f;
+	char *v;
+	int i;
+	int t;
+
+	static const twitter_filter_type_t types[] = {
+		TWITTER_FILTER_TYPE_FOLLOW,
+		TWITTER_FILTER_TYPE_TRACK
+	};
+
+	static const char *typestrs[] = {
+		"follow",
+		"track"
+	};
+
+	for (f = fs; *f; f++) {
+		if ((v = strchr(*f, ':')) == NULL)
+			continue;
+
+		*(v++) = 0;
+
+		for (t = -1, i = 0; i < G_N_ELEMENTS(types); i++) {
+			if (g_strcasecmp(typestrs[i], *f) == 0) {
+				t = i;
+				break;
+			}
+		}
+
+		if (t < 0 || strlen(v) == 0)
+			continue;
+
+		tf = twitter_filter_get(c, types[t], v);
+		ret = g_slist_prepend(ret, tf);
+	}
+
+	g_strfreev(fs);
+	return ret;
+}
+
 /**
  * Main loop function
  */
@@ -435,7 +644,11 @@ static void twitter_logout(struct im_connection *ic)
 		imcb_chat_free(td->timeline_gc);
 
 	if (td) {
+		if (td->filter_update_id > 0)
+			b_event_remove(td->filter_update_id);
+
 		http_close(td->stream);
+		twitter_filter_remove_all(ic);
 		oauth_info_free(td->oauth_info);
 		g_free(td->user);
 		g_free(td->prefix);
@@ -508,12 +721,57 @@ static void twitter_chat_invite(struct groupchat *c, char *who, char *message)
 {
 }
 
+static struct groupchat *twitter_chat_join(struct im_connection *ic,
+                                           const char *room, const char *nick,
+                                           const char *password, set_t **sets)
+{
+	struct groupchat *c = imcb_chat_new(ic, room);
+	GSList *fs = twitter_filter_parse(c, room);
+	GString *topic = g_string_new("");
+	struct twitter_filter *tf;
+	GSList *l;
+
+	fs = g_slist_sort(fs, (GCompareFunc) twitter_filter_cmp);
+
+	for (l = fs; l; l = g_slist_next(l)) {
+		tf = l->data;
+
+		if (topic->len > 0)
+			g_string_append(topic, ", ");
+
+		if (tf->type == TWITTER_FILTER_TYPE_FOLLOW)
+			g_string_append_c(topic, '@');
+
+		g_string_append(topic, tf->text);
+	}
+
+	if (topic->len > 0)
+		g_string_prepend(topic, "Twitter Filter: ");
+
+	imcb_chat_topic(c, NULL, topic->str, 0);
+	imcb_chat_add_buddy(c, ic->acc->user);
+
+	if (topic->len == 0) {
+		imcb_error(ic, "Failed to handle any filters");
+		imcb_chat_free(c);
+		c = NULL;
+	}
+
+	g_string_free(topic, TRUE);
+	g_slist_free(fs);
+
+	return c;
+}
+
 static void twitter_chat_leave(struct groupchat *c)
 {
 	struct twitter_data *td = c->ic->proto_data;
 
-	if (c != td->timeline_gc)
-		return;		/* WTF? */
+	if (c != td->timeline_gc) {
+		twitter_filter_remove(c);
+		imcb_chat_free(c);
+		return;
+	}
 
 	/* If the user leaves the channel: Fine. Rejoin him/her once new
 	   tweets come in. */
@@ -747,6 +1005,7 @@ void twitter_initmodule()
 	ret->remove_buddy = twitter_remove_buddy;
 	ret->chat_msg = twitter_chat_msg;
 	ret->chat_invite = twitter_chat_invite;
+	ret->chat_join = twitter_chat_join;
 	ret->chat_leave = twitter_chat_leave;
 	ret->keepalive = twitter_keepalive;
 	ret->add_permit = twitter_add_permit;
diff --git a/protocols/twitter/twitter.h b/protocols/twitter/twitter.h
index 8792b7c9..00230cc0 100644
--- a/protocols/twitter/twitter.h
+++ b/protocols/twitter/twitter.h
@@ -44,6 +44,12 @@ typedef enum
 	TWITTER_GOT_MENTIONS   = 0x40000,
 } twitter_flags_t;
 
+typedef enum
+{
+	TWITTER_FILTER_TYPE_FOLLOW = 0,
+	TWITTER_FILTER_TYPE_TRACK
+} twitter_filter_type_t;
+
 struct twitter_log_data;
 
 struct twitter_data
@@ -57,10 +63,13 @@ struct twitter_data
 	guint64 timeline_id;
 
 	GSList *follow_ids;
+	GSList *filters;
 	
 	guint64 last_status_id; /* For undo */
 	gint main_loop_id;
+	gint filter_update_id;
 	struct http_request *stream;
+	struct http_request *filter_stream;
 	struct groupchat *timeline_gc;
 	gint http_fails;
 	twitter_flags_t flags;
@@ -78,6 +87,15 @@ struct twitter_data
 	int log_id;
 };
 
+#define TWITTER_FILTER_UPDATE_WAIT 3000
+struct twitter_filter
+{
+	twitter_filter_type_t type;
+	char *text;
+	guint64 uid;
+	GSList *groupchats;
+};
+
 struct twitter_user_data
 {
 	guint64 last_id;
diff --git a/protocols/twitter/twitter_lib.c b/protocols/twitter/twitter_lib.c
index 718867a7..c8956606 100644
--- a/protocols/twitter/twitter_lib.c
+++ b/protocols/twitter/twitter_lib.c
@@ -50,6 +50,7 @@ struct twitter_xml_list {
 };
 
 struct twitter_xml_user {
+	guint64 uid;
 	char *name;
 	char *screen_name;
 };
@@ -60,6 +61,7 @@ struct twitter_xml_status {
 	struct twitter_xml_user *user;
 	guint64 id, rt_id; /* Usually equal, with RTs id == *original* id */
 	guint64 reply_to;
+	gboolean from_filter;
 };
 
 /**
@@ -391,11 +393,15 @@ static void twitter_http_get_users_lookup(struct http_request *req)
 struct twitter_xml_user *twitter_xt_get_user(const json_value *node)
 {
 	struct twitter_xml_user *txu;
+	json_value *jv;
 	
 	txu = g_new0(struct twitter_xml_user, 1);
 	txu->name = g_strdup(json_o_str(node, "name"));
 	txu->screen_name = g_strdup(json_o_str(node, "screen_name"));
 	
+	jv = json_o_get(node, "id");
+	txu->uid = jv->u.integer;
+	
 	return txu;
 }
 
@@ -656,6 +662,44 @@ static char *twitter_msg_add_id(struct im_connection *ic,
 }
 
 /**
+ * Function that is called to see the filter statuses in groupchat windows.
+ */
+static void twitter_status_show_filter(struct im_connection *ic, struct twitter_xml_status *status)
+{
+	struct twitter_data *td = ic->proto_data;
+	char *msg = twitter_msg_add_id(ic, status, "");
+	struct twitter_filter *tf;
+	GSList *f;
+	GSList *l;
+
+	for (f = td->filters; f; f = g_slist_next(f)) {
+		tf = f->data;
+
+		switch (tf->type) {
+		case TWITTER_FILTER_TYPE_FOLLOW:
+			if (status->user->uid != tf->uid)
+				continue;
+			break;
+
+		case TWITTER_FILTER_TYPE_TRACK:
+			if (strcasestr(status->text, tf->text) == NULL)
+				continue;
+			break;
+
+		default:
+			continue;
+		}
+
+		for (l = tf->groupchats; l; l = g_slist_next(l)) {
+			imcb_chat_msg(l->data, status->user->screen_name,
+				      msg ? msg : status->text, 0, 0);
+		}
+	}
+
+	g_free(msg);
+}
+
+/**
  * Function that is called to see the statuses in a groupchat window.
  */
 static void twitter_status_show_chat(struct im_connection *ic, struct twitter_xml_status *status)
@@ -730,7 +774,9 @@ static void twitter_status_show(struct im_connection *ic, struct twitter_xml_sta
 	if (set_getbool(&ic->acc->set, "strip_newlines"))
 		strip_newlines(status->text);
 	
-	if (td->flags & TWITTER_MODE_CHAT)
+	if (status->from_filter)
+		twitter_status_show_filter(ic, status);
+	else if (td->flags & TWITTER_MODE_CHAT)
 		twitter_status_show_chat(ic, status);
 	else
 		twitter_status_show_msg(ic, status);
@@ -744,7 +790,7 @@ static void twitter_status_show(struct im_connection *ic, struct twitter_xml_sta
 	g_free(last_id_str);
 }
 
-static gboolean twitter_stream_handle_object(struct im_connection *ic, json_value *o);
+static gboolean twitter_stream_handle_object(struct im_connection *ic, json_value *o, gboolean from_filter);
 
 static void twitter_http_stream(struct http_request *req)
 {
@@ -753,6 +799,7 @@ static void twitter_http_stream(struct http_request *req)
 	json_value *parsed;
 	int len = 0;
 	char c, *nl;
+	gboolean from_filter;
 	
 	if (!g_slist_find(twitter_connections, ic))
 		return;
@@ -761,7 +808,11 @@ static void twitter_http_stream(struct http_request *req)
 	td = ic->proto_data;
 	
 	if ((req->flags & HTTPC_EOF) || !req->reply_body) {
-		td->stream = NULL;
+		if (req == td->stream)
+			td->stream = NULL;
+		else if (req == td->filter_stream)
+			td->filter_stream = NULL;
+
 		imcb_error(ic, "Stream closed (%s)", req->status_string);
 		imc_logout(ic, TRUE);
 		return;
@@ -778,7 +829,8 @@ static void twitter_http_stream(struct http_request *req)
 		req->reply_body[len] = '\0';
 		
 		if ((parsed = json_parse(req->reply_body, req->body_size))) {
-			twitter_stream_handle_object(ic, parsed);
+			from_filter = (req == td->filter_stream);
+			twitter_stream_handle_object(ic, parsed, from_filter);
 		}
 		json_value_free(parsed);
 		req->reply_body[len] = c;
@@ -794,13 +846,14 @@ static void twitter_http_stream(struct http_request *req)
 static gboolean twitter_stream_handle_event(struct im_connection *ic, json_value *o);
 static gboolean twitter_stream_handle_status(struct im_connection *ic, struct twitter_xml_status *txs);
 
-static gboolean twitter_stream_handle_object(struct im_connection *ic, json_value *o)
+static gboolean twitter_stream_handle_object(struct im_connection *ic, json_value *o, gboolean from_filter)
 {
 	struct twitter_data *td = ic->proto_data;
 	struct twitter_xml_status *txs;
 	json_value *c;
 	
 	if ((txs = twitter_xt_get_status(o))) {
+		txs->from_filter = from_filter;
 		gboolean ret = twitter_stream_handle_status(ic, txs);
 		txs_free(txs);
 		return ret;
@@ -898,6 +951,169 @@ gboolean twitter_open_stream(struct im_connection *ic)
 	return FALSE;
 }
 
+static gboolean twitter_filter_stream(struct im_connection *ic)
+{
+	struct twitter_data *td = ic->proto_data;
+	char *args[4] = {"follow", NULL, "track", NULL};
+	GString *followstr = g_string_new("");
+	GString *trackstr = g_string_new("");
+	gboolean ret = FALSE;
+	struct twitter_filter *tf;
+	GSList *l;
+
+	for (l = td->filters; l; l = g_slist_next(l)) {
+		tf = l->data;
+
+		switch (tf->type) {
+		case TWITTER_FILTER_TYPE_FOLLOW:
+			if (followstr->len > 0)
+				g_string_append_c(followstr, ',');
+
+			g_string_append_printf(followstr, "%" G_GUINT64_FORMAT,
+					       tf->uid);
+			break;
+
+		case TWITTER_FILTER_TYPE_TRACK:
+			if (trackstr->len > 0)
+				g_string_append_c(trackstr, ',');
+
+			g_string_append(trackstr, tf->text);
+			break;
+
+		default:
+			continue;
+		}
+	}
+
+	args[1] = followstr->str;
+	args[3] = trackstr->str;
+
+	if (td->filter_stream)
+		http_close(td->filter_stream);
+
+	if ((td->filter_stream = twitter_http(ic, TWITTER_FILTER_STREAM_URL,
+	                                      twitter_http_stream, ic, 0,
+					      args, 4))) {
+		/* This flag must be enabled or we'll get no data until EOF
+		   (which err, kind of, defeats the purpose of a streaming API). */
+		td->filter_stream->flags |= HTTPC_STREAMING;
+		ret = TRUE;
+	}
+
+	g_string_free(followstr, TRUE);
+	g_string_free(trackstr, TRUE);
+
+	return ret;
+}
+
+static void twitter_filter_users_post(struct http_request *req)
+{
+	struct im_connection *ic = req->data;
+	struct twitter_data *td;
+	struct twitter_filter *tf;
+	GList *users = NULL;
+	json_value *parsed;
+	json_value *id;
+	const char *name;
+	GString *fstr;
+	GSList *l;
+	GList *u;
+	int i;
+
+	// Check if the connection is still active.
+	if (!g_slist_find(twitter_connections, ic))
+		return;
+
+	td = ic->proto_data;
+
+	if (!(parsed = twitter_parse_response(ic, req)))
+		return;
+
+	for (l = td->filters; l; l = g_slist_next(l)) {
+		tf = l->data;
+
+		if (tf->type == TWITTER_FILTER_TYPE_FOLLOW)
+			users = g_list_prepend(users, tf);
+	}
+
+	if (parsed->type != json_array)
+		goto finish;
+
+	for (i = 0; i < parsed->u.array.length; i++) {
+		id = json_o_get(parsed->u.array.values[i], "id");
+		name = json_o_str(parsed->u.array.values[i], "screen_name");
+
+		if (!name || !id || id->type != json_integer)
+			continue;
+
+		for (u = users; u; u = g_list_next(u)) {
+			tf = u->data;
+
+			if (g_strcasecmp(tf->text, name) == 0) {
+				tf->uid = id->u.integer;
+				users = g_list_delete_link(users, u);
+				break;
+			}
+		}
+	}
+
+finish:
+	json_value_free(parsed);
+	twitter_filter_stream(ic);
+
+	if (!users)
+		return;
+
+	fstr = g_string_new("");
+
+	for (u = users; u; u = g_list_next(u)) {
+		if (fstr->len > 0)
+			g_string_append(fstr, ", ");
+
+		g_string_append(fstr, tf->text);
+	}
+
+	imcb_error(ic, "Failed UID acquisitions: %s", fstr->str);
+
+	g_string_free(fstr, TRUE);
+	g_list_free(users);
+}
+
+gboolean twitter_open_filter_stream(struct im_connection *ic)
+{
+	struct twitter_data *td = ic->proto_data;
+	char *args[2] = {"screen_name", NULL};
+	GString *ustr = g_string_new("");
+	struct twitter_filter *tf;
+	struct http_request *req;
+	GSList *l;
+
+	for (l = td->filters; l; l = g_slist_next(l)) {
+		tf = l->data;
+
+		if (tf->type != TWITTER_FILTER_TYPE_FOLLOW || tf->uid != 0)
+			continue;
+
+		if (ustr->len > 0)
+			g_string_append_c(ustr, ',');
+
+		g_string_append(ustr, tf->text);
+	}
+
+	if (ustr->len == 0) {
+		g_string_free(ustr, TRUE);
+		return twitter_filter_stream(ic);
+	}
+
+	args[1] = ustr->str;
+	req = twitter_http(ic, TWITTER_USERS_LOOKUP_URL,
+			   twitter_filter_users_post,
+			   ic, 0, args, 2);
+
+	g_string_free(ustr, TRUE);
+	return req != NULL;
+}
+
 static void twitter_get_home_timeline(struct im_connection *ic, gint64 next_cursor);
 static void twitter_get_mentions(struct im_connection *ic, gint64 next_cursor);
 
diff --git a/protocols/twitter/twitter_lib.h b/protocols/twitter/twitter_lib.h
index 7fd3b808..ee103100 100644
--- a/protocols/twitter/twitter_lib.h
+++ b/protocols/twitter/twitter_lib.h
@@ -78,9 +78,12 @@
 /* Report spam */
 #define TWITTER_REPORT_SPAM_URL "/users/report_spam.json"
 
+/* Stream URLs */
 #define TWITTER_USER_STREAM_URL "https://userstream.twitter.com/1.1/user.json"
+#define TWITTER_FILTER_STREAM_URL "https://stream.twitter.com/1.1/statuses/filter.json"
 
 gboolean twitter_open_stream(struct im_connection *ic);
+gboolean twitter_open_filter_stream(struct im_connection *ic);
 gboolean twitter_get_timeline(struct im_connection *ic, gint64 next_cursor);
 void twitter_get_friends_ids(struct im_connection *ic, gint64 next_cursor);
 void twitter_get_statuses_friends(struct im_connection *ic, gint64 next_cursor);