libceph: mon_get_version request infrastructure

Add support for mon_get_version requests to libceph.  This reuses much
of the ceph_mon_generic_request infrastructure, with one exception.
Older OSDs don't set mon_get_version reply hdr->tid even if the
original request had a non-zero tid, which makes it impossible to
lookup ceph_mon_generic_request contexts by tid in get_generic_reply()
for such replies.  As a workaround, we allocate a reply message on the
reply path.  This can probably interfere with revoke, but I don't see
a better way.

Signed-off-by: Ilya Dryomov <ilya.dryomov@inktank.com>
Reviewed-by: Sage Weil <sage@inktank.com>

diff --git a/include/linux/ceph/mon_client.h b/include/linux/ceph/mon_client.h
index a486f390dfbeeb9d4004921bb973852bd586a808..585ef9450e9d21258192880c80ae47fda3960bfa 100644 (file)
--- a/include/linux/ceph/mon_client.h
+++ b/include/linux/ceph/mon_client.h
@@ -40,9 +40,9 @@ struct ceph_mon_request {
 };
 
 /*
- * ceph_mon_generic_request is being used for the statfs and poolop requests
- * which are bening done a bit differently because we need to get data back
- * to the caller
+ * ceph_mon_generic_request is being used for the statfs, poolop and
+ * mon_get_version requests which are being done a bit differently
+ * because we need to get data back to the caller
  */
 struct ceph_mon_generic_request {
        struct kref kref;
@@ -108,6 +108,9 @@ extern void ceph_monc_request_next_osdmap(struct ceph_mon_client *monc);
 extern int ceph_monc_do_statfs(struct ceph_mon_client *monc,
                               struct ceph_statfs *buf);
 
+extern int ceph_monc_do_get_version(struct ceph_mon_client *monc,
+                                   const char *what, u64 *newest);
+
 extern int ceph_monc_open_session(struct ceph_mon_client *monc);
 
 extern int ceph_monc_validate_auth(struct ceph_mon_client *monc);

diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c
index 67d7721d237e1638c21bc62da98f7275a9991026..1675021d8c12cf6e941720802553f942171eb79b 100644 (file)
--- a/net/ceph/ceph_common.c
+++ b/net/ceph/ceph_common.c
@@ -72,6 +72,8 @@ const char *ceph_msg_type_name(int type)
        case CEPH_MSG_MON_SUBSCRIBE_ACK: return "mon_subscribe_ack";
        case CEPH_MSG_STATFS: return "statfs";
        case CEPH_MSG_STATFS_REPLY: return "statfs_reply";
+       case CEPH_MSG_MON_GET_VERSION: return "mon_get_version";
+       case CEPH_MSG_MON_GET_VERSION_REPLY: return "mon_get_version_reply";
        case CEPH_MSG_MDS_MAP: return "mds_map";
        case CEPH_MSG_CLIENT_SESSION: return "client_session";
        case CEPH_MSG_CLIENT_RECONNECT: return "client_reconnect";

diff --git a/net/ceph/debugfs.c b/net/ceph/debugfs.c
index 8903dcee8d8e79acb898f169e2aa05319c6f7233..d1a62c69a9f4fd1cff25ca458a66d8e2c6093099 100644 (file)
--- a/net/ceph/debugfs.c
+++ b/net/ceph/debugfs.c
@@ -129,6 +129,8 @@ static int monc_show(struct seq_file *s, void *p)
                        seq_printf(s, "%llu statfs\n", req->tid);
                else if (op == CEPH_MSG_POOLOP)
                        seq_printf(s, "%llu poolop\n", req->tid);
+               else if (op == CEPH_MSG_MON_GET_VERSION)
+                       seq_printf(s, "%llu mon_get_version", req->tid);
                else
                        seq_printf(s, "%llu unknown\n", req->tid);
        }

diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c
index 2ac9ef35110b3e9ea07363d6f2d3c0eb2e4ee27b..11d8d2f2708ac2abbcf9661e80434a3eebc350d8 100644 (file)
--- a/net/ceph/mon_client.c
+++ b/net/ceph/mon_client.c
@@ -477,14 +477,13 @@ static struct ceph_msg *get_generic_reply(struct ceph_connection *con,
        return m;
 }
 
-static int do_generic_request(struct ceph_mon_client *monc,
-                             struct ceph_mon_generic_request *req)
+static int __do_generic_request(struct ceph_mon_client *monc, u64 tid,
+                               struct ceph_mon_generic_request *req)
 {
        int err;
 
        /* register request */
-       mutex_lock(&monc->mutex);
-       req->tid = ++monc->last_tid;
+       req->tid = tid != 0 ? tid : ++monc->last_tid;
        req->request->hdr.tid = cpu_to_le64(req->tid);
        __insert_generic_request(monc, req);
        monc->num_generic_requests++;
@@ -496,13 +495,24 @@ static int do_generic_request(struct ceph_mon_client *monc,
        mutex_lock(&monc->mutex);
        rb_erase(&req->node, &monc->generic_request_tree);
        monc->num_generic_requests--;
-       mutex_unlock(&monc->mutex);
 
        if (!err)
                err = req->result;
        return err;
 }
 
+static int do_generic_request(struct ceph_mon_client *monc,
+                             struct ceph_mon_generic_request *req)
+{
+       int err;
+
+       mutex_lock(&monc->mutex);
+       err = __do_generic_request(monc, 0, req);
+       mutex_unlock(&monc->mutex);
+
+       return err;
+}
+
 /*
  * statfs
  */
@@ -579,6 +589,96 @@ int ceph_monc_do_statfs(struct ceph_mon_client *monc, struct ceph_statfs *buf)
 }
 EXPORT_SYMBOL(ceph_monc_do_statfs);
 
+static void handle_get_version_reply(struct ceph_mon_client *monc,
+                                    struct ceph_msg *msg)
+{
+       struct ceph_mon_generic_request *req;
+       u64 tid = le64_to_cpu(msg->hdr.tid);
+       void *p = msg->front.iov_base;
+       void *end = p + msg->front_alloc_len;
+       u64 handle;
+
+       dout("%s %p tid %llu\n", __func__, msg, tid);
+
+       ceph_decode_need(&p, end, 2*sizeof(u64), bad);
+       handle = ceph_decode_64(&p);
+       if (tid != 0 && tid != handle)
+               goto bad;
+
+       mutex_lock(&monc->mutex);
+       req = __lookup_generic_req(monc, handle);
+       if (req) {
+               *(u64 *)req->buf = ceph_decode_64(&p);
+               req->result = 0;
+               get_generic_request(req);
+       }
+       mutex_unlock(&monc->mutex);
+       if (req) {
+               complete_all(&req->completion);
+               put_generic_request(req);
+       }
+
+       return;
+bad:
+       pr_err("corrupt mon_get_version reply\n");
+       ceph_msg_dump(msg);
+}
+
+/*
+ * Send MMonGetVersion and wait for the reply.
+ *
+ * @what: one of "mdsmap", "osdmap" or "monmap"
+ */
+int ceph_monc_do_get_version(struct ceph_mon_client *monc, const char *what,
+                            u64 *newest)
+{
+       struct ceph_mon_generic_request *req;
+       void *p, *end;
+       u64 tid;
+       int err;
+
+       req = kzalloc(sizeof(*req), GFP_NOFS);
+       if (!req)
+               return -ENOMEM;
+
+       kref_init(&req->kref);
+       req->buf = newest;
+       req->buf_len = sizeof(*newest);
+       init_completion(&req->completion);
+
+       req->request = ceph_msg_new(CEPH_MSG_MON_GET_VERSION,
+                                   sizeof(u64) + sizeof(u32) + strlen(what),
+                                   GFP_NOFS, true);
+       if (!req->request) {
+               err = -ENOMEM;
+               goto out;
+       }
+
+       req->reply = ceph_msg_new(CEPH_MSG_MON_GET_VERSION_REPLY, 1024,
+                                 GFP_NOFS, true);
+       if (!req->reply) {
+               err = -ENOMEM;
+               goto out;
+       }
+
+       p = req->request->front.iov_base;
+       end = p + req->request->front_alloc_len;
+
+       /* fill out request */
+       mutex_lock(&monc->mutex);
+       tid = ++monc->last_tid;
+       ceph_encode_64(&p, tid); /* handle */
+       ceph_encode_string(&p, end, what, strlen(what));
+
+       err = __do_generic_request(monc, tid, req);
+
+       mutex_unlock(&monc->mutex);
+out:
+       kref_put(&req->kref, release_generic_request);
+       return err;
+}
+EXPORT_SYMBOL(ceph_monc_do_get_version);
+
 /*
  * pool ops
  */
@@ -981,6 +1081,10 @@ static void dispatch(struct ceph_connection *con, struct ceph_msg *msg)
                handle_statfs_reply(monc, msg);
                break;
 
+       case CEPH_MSG_MON_GET_VERSION_REPLY:
+               handle_get_version_reply(monc, msg);
+               break;
+
        case CEPH_MSG_POOLOP_REPLY:
                handle_poolop_reply(monc, msg);
                break;
@@ -1029,6 +1133,15 @@ static struct ceph_msg *mon_alloc_msg(struct ceph_connection *con,
        case CEPH_MSG_AUTH_REPLY:
                m = ceph_msg_get(monc->m_auth_reply);
                break;
+       case CEPH_MSG_MON_GET_VERSION_REPLY:
+               if (le64_to_cpu(hdr->tid) != 0)
+                       return get_generic_reply(con, hdr, skip);
+
+               /*
+                * Older OSDs don't set reply tid even if the orignal
+                * request had a non-zero tid.  Workaround this weirdness
+                * by falling through to the allocate case.
+                */
        case CEPH_MSG_MON_MAP:
        case CEPH_MSG_MDS_MAP:
        case CEPH_MSG_OSD_MAP: