From e5e372da9a469dfe3ece40277090a7056c566838 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 22 May 2012 11:41:43 -0500 Subject: libceph: eliminate connection state "DEAD" The ceph connection state "DEAD" is never set and is therefore not needed. Eliminate it. Signed-off-by: Alex Elder Reviewed-by: Yehuda Sadeh --- include/linux/ceph/messenger.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 2521a95fa6d9..aa506cadea67 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -119,7 +119,6 @@ struct ceph_msg_pos { #define CLOSED 10 /* we've closed the connection */ #define SOCK_CLOSED 11 /* socket state changed to closed */ #define OPENING 13 /* open connection w/ (possibly new) peer */ -#define DEAD 14 /* dead, about to kfree */ #define BACKOFF 15 /* -- cgit v1.2.3 From 6384bb8b8e88a9c6bf2ae0d9517c2c0199177c34 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 29 May 2012 21:47:38 -0500 Subject: libceph: kill bad_proto ceph connection op No code sets a bad_proto method in its ceph connection operations vector, so just get rid of it. Signed-off-by: Alex Elder Reviewed-by: Yehuda Sadeh --- include/linux/ceph/messenger.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index aa506cadea67..74f6c9bd8074 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -31,9 +31,6 @@ struct ceph_connection_operations { int (*verify_authorizer_reply) (struct ceph_connection *con, int len); int (*invalidate_authorizer)(struct ceph_connection *con); - /* protocol version mismatch */ - void (*bad_proto) (struct ceph_connection *con); - /* there was some error on the socket (disconnect, whatever) */ void (*fault) (struct ceph_connection *con); -- cgit v1.2.3 From 15d9882c336db2db73ccf9871ae2398e452f694c Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Sat, 26 May 2012 23:26:43 -0500 Subject: libceph: embed ceph messenger structure in ceph_client A ceph client has a pointer to a ceph messenger structure in it. There is always exactly one ceph messenger for a ceph client, so there is no need to allocate it separate from the ceph client structure. Switch the ceph_client structure to embed its ceph_messenger structure. Signed-off-by: Alex Elder Reviewed-by: Yehuda Sadeh Reviewed-by: Sage Weil --- include/linux/ceph/libceph.h | 2 +- include/linux/ceph/messenger.h | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index 92eef7c3d3c5..927361c4b0a8 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -131,7 +131,7 @@ struct ceph_client { u32 supported_features; u32 required_features; - struct ceph_messenger *msgr; /* messenger instance */ + struct ceph_messenger msgr; /* messenger instance */ struct ceph_mon_client monc; struct ceph_osd_client osdc; diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 74f6c9bd8074..3fbd4be804ed 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -211,10 +211,11 @@ extern int ceph_msgr_init(void); extern void ceph_msgr_exit(void); extern void ceph_msgr_flush(void); -extern struct ceph_messenger *ceph_messenger_create( - struct ceph_entity_addr *myaddr, - u32 features, u32 required); -extern void ceph_messenger_destroy(struct ceph_messenger *); +extern void ceph_messenger_init(struct ceph_messenger *msgr, + struct ceph_entity_addr *myaddr, + u32 supported_features, + u32 required_features, + bool nocrc); extern void ceph_con_init(struct ceph_messenger *msgr, struct ceph_connection *con); -- cgit v1.2.3 From 928443cd9644e7cfd46f687dbeffda2d1a357ff9 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 22 May 2012 11:41:43 -0500 Subject: libceph: start separating connection flags from state A ceph_connection holds a mixture of connection state (as in "state machine" state) and connection flags in a single "state" field. To make the distinction more clear, define a new "flags" field and use it rather than the "state" field to hold Boolean flag values. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 3fbd4be804ed..920235e114ad 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -103,20 +103,25 @@ struct ceph_msg_pos { #define MAX_DELAY_INTERVAL (5 * 60 * HZ) /* - * ceph_connection state bit flags + * ceph_connection flag bits */ + #define LOSSYTX 0 /* we can close channel or drop messages on errors */ -#define CONNECTING 1 -#define NEGOTIATING 2 #define KEEPALIVE_PENDING 3 #define WRITE_PENDING 4 /* we have data ready to send */ +#define SOCK_CLOSED 11 /* socket state changed to closed */ +#define BACKOFF 15 + +/* + * ceph_connection states + */ +#define CONNECTING 1 +#define NEGOTIATING 2 #define STANDBY 8 /* no outgoing messages, socket closed. we keep * the ceph_connection around to maintain shared * state with the peer. */ #define CLOSED 10 /* we've closed the connection */ -#define SOCK_CLOSED 11 /* socket state changed to closed */ #define OPENING 13 /* open connection w/ (possibly new) peer */ -#define BACKOFF 15 /* * A single connection with another host. @@ -133,7 +138,8 @@ struct ceph_connection { struct ceph_messenger *msgr; struct socket *sock; - unsigned long state; /* connection state (see flags above) */ + unsigned long flags; + unsigned long state; const char *error_msg; /* error message, if any */ struct ceph_entity_addr peer_addr; /* peer address */ -- cgit v1.2.3 From ce2c8903e76e690846a00a0284e4bd9ee954d680 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 22 May 2012 22:15:49 -0500 Subject: libceph: start tracking connection socket state Start explicitly keeping track of the state of a ceph connection's socket, separate from the state of the connection itself. Create placeholder functions to encapsulate the state transitions. -------- | NEW* | transient initial state -------- | con_sock_state_init() v ---------- | CLOSED | initialized, but no socket (and no ---------- TCP connection) ^ \ | \ con_sock_state_connecting() | ---------------------- | \ + con_sock_state_closed() \ |\ \ | \ \ | ----------- \ | | CLOSING | socket event; \ | ----------- await close \ | ^ | | | | | + con_sock_state_closing() | | / \ | | / --------------- | | / \ v | / -------------- | / -----------------| CONNECTING | socket created, TCP | | / -------------- connect initiated | | | con_sock_state_connected() | | v ------------- | CONNECTED | TCP connection established ------------- Make the socket state an atomic variable, reinforcing that it's a distinct transtion with no possible "intermediate/both" states. This is almost certainly overkill at this point, though the transitions into CONNECTED and CLOSING state do get called via socket callback (the rest of the transitions occur with the connection mutex held). We can back out the atomicity later. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 920235e114ad..5e852f444f68 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -137,14 +137,18 @@ struct ceph_connection { const struct ceph_connection_operations *ops; struct ceph_messenger *msgr; + + atomic_t sock_state; struct socket *sock; + struct ceph_entity_addr peer_addr; /* peer address */ + struct ceph_entity_addr peer_addr_for_me; + unsigned long flags; unsigned long state; const char *error_msg; /* error message, if any */ - struct ceph_entity_addr peer_addr; /* peer address */ struct ceph_entity_name peer_name; /* peer name */ - struct ceph_entity_addr peer_addr_for_me; + unsigned peer_features; u32 connect_seq; /* identify the most recent connection attempt for this connection, client */ -- cgit v1.2.3 From 67130934fb579fdf0f2f6d745960264378b57dc8 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Sat, 26 May 2012 23:26:43 -0500 Subject: libceph: embed ceph connection structure in mon_client A monitor client has a pointer to a ceph connection structure in it. This is the only one of the three ceph client types that do it this way; the OSD and MDS clients embed the connection into their main structures. There is always exactly one ceph connection for a monitor client, so there is no need to allocate it separate from the monitor client structure. So switch the ceph_mon_client structure to embed its ceph_connection structure. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/mon_client.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/mon_client.h b/include/linux/ceph/mon_client.h index 545f85917780..2113e3850a4e 100644 --- a/include/linux/ceph/mon_client.h +++ b/include/linux/ceph/mon_client.h @@ -70,7 +70,7 @@ struct ceph_mon_client { bool hunting; int cur_mon; /* last monitor i contacted */ unsigned long sub_sent, sub_renew_after; - struct ceph_connection *con; + struct ceph_connection con; bool have_fsid; /* pending generic requests */ -- cgit v1.2.3 From 1bfd89f4e6e1adc6a782d94aa5d4c53be1e404d7 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Sat, 26 May 2012 23:26:43 -0500 Subject: libceph: fully initialize connection in con_init() Move the initialization of a ceph connection's private pointer, operations vector pointer, and peer name information into ceph_con_init(). Rearrange the arguments so the connection pointer is first. Hide the byte-swapping of the peer entity number inside ceph_con_init() Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 5e852f444f68..dd27837f79ac 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -227,8 +227,10 @@ extern void ceph_messenger_init(struct ceph_messenger *msgr, u32 required_features, bool nocrc); -extern void ceph_con_init(struct ceph_messenger *msgr, - struct ceph_connection *con); +extern void ceph_con_init(struct ceph_connection *con, void *private, + const struct ceph_connection_operations *ops, + struct ceph_messenger *msgr, __u8 entity_type, + __u64 entity_num); extern void ceph_con_open(struct ceph_connection *con, struct ceph_entity_addr *addr); extern bool ceph_con_opened(struct ceph_connection *con); -- cgit v1.2.3 From 38941f8031bf042dba3ced6394ba3a3b16c244ea Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Fri, 1 Jun 2012 14:56:43 -0500 Subject: libceph: have messages point to their connection When a ceph message is queued for sending it is placed on a list of pending messages (ceph_connection->out_queue). When they are actually sent over the wire, they are moved from that list to another (ceph_connection->out_sent). When acknowledgement for the message is received, it is removed from the sent messages list. During that entire time the message is "in the possession" of a single ceph connection. Keep track of that connection in the message. This will be used in the next patch (and is a helpful bit of information for debugging anyway). Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index dd27837f79ac..6df837f72761 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -77,7 +77,10 @@ struct ceph_msg { unsigned nr_pages; /* size of page array */ unsigned page_alignment; /* io offset in first page */ struct ceph_pagelist *pagelist; /* instead of pages */ + + struct ceph_connection *con; struct list_head list_head; + struct kref kref; struct bio *bio; /* instead of pages/pagelist */ struct bio *bio_iter; /* bio iterator */ -- cgit v1.2.3 From 6740a845b2543cc46e1902ba21bac743fbadd0dc Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Fri, 1 Jun 2012 14:56:43 -0500 Subject: libceph: make ceph_con_revoke() a msg operation ceph_con_revoke() is passed both a message and a ceph connection. Now that any message associated with a connection holds a pointer to that connection, there's no need to provide the connection when revoking a message. This has the added benefit of precluding the possibility of the providing the wrong connection pointer. If the message's connection pointer is null, it is not being tracked by any connection, so revoking it is a no-op. This is supported as a convenience for upper layers, so they can revoke a message that is not actually "in flight." Rename the function ceph_msg_revoke() to reflect that it is really an operation on a message, not a connection. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 6df837f72761..9008f81c20cd 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -239,7 +239,8 @@ extern void ceph_con_open(struct ceph_connection *con, extern bool ceph_con_opened(struct ceph_connection *con); extern void ceph_con_close(struct ceph_connection *con); extern void ceph_con_send(struct ceph_connection *con, struct ceph_msg *msg); -extern void ceph_con_revoke(struct ceph_connection *con, struct ceph_msg *msg); + +extern void ceph_msg_revoke(struct ceph_msg *msg); extern void ceph_con_revoke_message(struct ceph_connection *con, struct ceph_msg *msg); extern void ceph_con_keepalive(struct ceph_connection *con); -- cgit v1.2.3 From 8921d114f5574c6da2cdd00749d185633ecf88f3 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Fri, 1 Jun 2012 14:56:43 -0500 Subject: libceph: make ceph_con_revoke_message() a msg op ceph_con_revoke_message() is passed both a message and a ceph connection. A ceph_msg allocated for incoming messages on a connection always has a pointer to that connection, so there's no need to provide the connection when revoking such a message. Note that the existing logic does not preclude the message supplied being a null/bogus message pointer. The only user of this interface is the OSD client, and the only value an osd client passes is a request's r_reply field. That is always non-null (except briefly in an error path in ceph_osdc_alloc_request(), and that drops the only reference so the request won't ever have a reply to revoke). So we can safely assume the passed-in message is non-null, but add a BUG_ON() to make it very obvious we are imposing this restriction. Rename the function ceph_msg_revoke_incoming() to reflect that it is really an operation on an incoming message. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 9008f81c20cd..a334dbd1b324 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -241,8 +241,8 @@ extern void ceph_con_close(struct ceph_connection *con); extern void ceph_con_send(struct ceph_connection *con, struct ceph_msg *msg); extern void ceph_msg_revoke(struct ceph_msg *msg); -extern void ceph_con_revoke_message(struct ceph_connection *con, - struct ceph_msg *msg); +extern void ceph_msg_revoke_incoming(struct ceph_msg *msg); + extern void ceph_con_keepalive(struct ceph_connection *con); extern struct ceph_connection *ceph_con_get(struct ceph_connection *con); extern void ceph_con_put(struct ceph_connection *con); -- cgit v1.2.3 From d59315ca8c0de00df9b363f94a2641a30961ca1c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 21 Jun 2012 12:49:23 -0700 Subject: libceph: drop ceph_con_get/put helpers and nref member These are no longer used. Every ceph_connection instance is embedded in another structure, and refcounts manipulated via the get/put ops. Signed-off-by: Sage Weil --- include/linux/ceph/messenger.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index a334dbd1b324..cc6f9bdcf466 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -135,7 +135,6 @@ struct ceph_msg_pos { */ struct ceph_connection { void *private; - atomic_t nref; const struct ceph_connection_operations *ops; -- cgit v1.2.3 From e27947c767f5bed15048f4e4dad3e2eb69133697 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Wed, 23 May 2012 14:35:23 -0500 Subject: libceph: define and use an explicit CONNECTED state There is no state explicitly defined when a ceph connection is fully operational. So define one. It's set when the connection sequence completes successfully, and is cleared when the connection gets closed. Be a little more careful when examining the old state when a socket disconnect event is reported. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index cc6f9bdcf466..002d504df3b7 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -120,6 +120,7 @@ struct ceph_msg_pos { */ #define CONNECTING 1 #define NEGOTIATING 2 +#define CONNECTED 5 #define STANDBY 8 /* no outgoing messages, socket closed. we keep * the ceph_connection around to maintain shared * state with the peer. */ -- cgit v1.2.3 From 261030215d970c62f799e6e508e3c68fc7ec2aa9 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Thu, 21 Jun 2012 12:49:23 -0700 Subject: libceph: drop declaration of ceph_con_get() For some reason the declaration of ceph_con_get() and ceph_con_put() did not get deleted in this commit: d59315ca libceph: drop ceph_con_get/put helpers and nref member Clean that up. Signed-off-by: Alex Elder --- include/linux/ceph/messenger.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 002d504df3b7..dd4ef1f8ec93 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -244,8 +244,6 @@ extern void ceph_msg_revoke(struct ceph_msg *msg); extern void ceph_msg_revoke_incoming(struct ceph_msg *msg); extern void ceph_con_keepalive(struct ceph_connection *con); -extern struct ceph_connection *ceph_con_get(struct ceph_connection *con); -extern void ceph_con_put(struct ceph_connection *con); extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags, bool can_fail); -- cgit v1.2.3 From b7a9e5dd40f17a48a72f249b8bbc989b63bae5fd Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 27 Jun 2012 12:24:08 -0700 Subject: libceph: set peer name on con_open, not init The peer name may change on each open attempt, even when the connection is reused. Signed-off-by: Sage Weil --- include/linux/ceph/messenger.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index dd4ef1f8ec93..478f814f2100 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -232,9 +232,9 @@ extern void ceph_messenger_init(struct ceph_messenger *msgr, extern void ceph_con_init(struct ceph_connection *con, void *private, const struct ceph_connection_operations *ops, - struct ceph_messenger *msgr, __u8 entity_type, - __u64 entity_num); + struct ceph_messenger *msgr); extern void ceph_con_open(struct ceph_connection *con, + __u8 entity_type, __u64 entity_num, struct ceph_entity_addr *addr); extern bool ceph_con_opened(struct ceph_connection *con); extern void ceph_con_close(struct ceph_connection *con); -- cgit v1.2.3 From d50b409fb8698571d8209e5adfe122e287e31290 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 9 Jul 2012 14:22:34 -0700 Subject: libceph: initialize msgpool message types Initialize the type field for messages in a msgpool. The caller was doing this for osd ops, but not for the reply messages. Reported-by: Alex Elder Signed-off-by: Sage Weil --- include/linux/ceph/msgpool.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/msgpool.h b/include/linux/ceph/msgpool.h index a362605f9368..09fa96b43436 100644 --- a/include/linux/ceph/msgpool.h +++ b/include/linux/ceph/msgpool.h @@ -11,10 +11,11 @@ struct ceph_msgpool { const char *name; mempool_t *pool; + int type; /* preallocated message type */ int front_len; /* preallocated payload size */ }; -extern int ceph_msgpool_init(struct ceph_msgpool *pool, +extern int ceph_msgpool_init(struct ceph_msgpool *pool, int type, int front_len, int size, bool blocking, const char *name); extern void ceph_msgpool_destroy(struct ceph_msgpool *pool); -- cgit v1.2.3 From a16cb1f70799c851410d9dca0a24122e258df06c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 10 Jul 2012 11:53:34 -0700 Subject: libceph: fix messenger retry In ancient times, the messenger could both initiate and accept connections. An artifact if that was data structures to store/process an incoming ceph_msg_connect request and send an outgoing ceph_msg_connect_reply. Sadly, the negotiation code was referencing those structures and ignoring important information (like the peer's connect_seq) from the correct ones. Among other things, this fixes tight reconnect loops where the server sends RETRY_SESSION and we (the client) retries with the same connect_seq as last time. This bug pretty easily triggered by injecting socket failures on the MDS and running some fs workload like workunits/direct_io/test_sync_io. Signed-off-by: Sage Weil --- include/linux/ceph/messenger.h | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 478f814f2100..cfb1bbdac624 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -172,16 +172,8 @@ struct ceph_connection { /* connection negotiation temps */ char in_banner[CEPH_BANNER_MAX_LEN]; - union { - struct { /* outgoing connection */ - struct ceph_msg_connect out_connect; - struct ceph_msg_connect_reply in_reply; - }; - struct { /* incoming */ - struct ceph_msg_connect in_connect; - struct ceph_msg_connect_reply out_reply; - }; - }; + struct ceph_msg_connect out_connect; + struct ceph_msg_connect_reply in_reply; struct ceph_entity_addr actual_peer_addr; /* message out temps */ -- cgit v1.2.3 From a2a3258417eb6a1799cf893350771428875a8287 Mon Sep 17 00:00:00 2001 From: Guanjun He Date: Sun, 8 Jul 2012 19:50:33 -0700 Subject: libceph: prevent the race of incoming work during teardown Add an atomic variable 'stopping' as flag in struct ceph_messenger, set this flag to 1 in function ceph_destroy_client(), and add the condition code in function ceph_data_ready() to test the flag value, if true(1), just return. Signed-off-by: Guanjun He Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index cfb1bbdac624..a310d7fe6e29 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -50,6 +50,7 @@ struct ceph_messenger { struct ceph_entity_inst inst; /* my name+address */ struct ceph_entity_addr my_enc_addr; + atomic_t stopping; bool nocrc; /* -- cgit v1.2.3 From c61a1abd215c1ccd6fa73104c79e79987ed3aa98 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 3 Jul 2012 16:01:18 -0500 Subject: libceph: fix off-by-one bug in ceph_encode_filepath() There is a BUG_ON() call that doesn't account for the single byte structure version at the start of an encoded filepath in ceph_encode_filepath(). Fix that. Signed-off-by: Alex Elder Reviewed-by: Yehuda Sadeh Reviewed-by: Josh Durgin --- include/linux/ceph/decode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/decode.h b/include/linux/ceph/decode.h index d8615dee5808..bcbd66c84890 100644 --- a/include/linux/ceph/decode.h +++ b/include/linux/ceph/decode.h @@ -151,7 +151,7 @@ static inline void ceph_encode_filepath(void **p, void *end, u64 ino, const char *path) { u32 len = path ? strlen(path) : 0; - BUG_ON(*p + sizeof(ino) + sizeof(len) + len > end); + BUG_ON(*p + 1 + sizeof(ino) + sizeof(len) + len > end); ceph_encode_8(p, 1); ceph_encode_64(p, ino); ceph_encode_32(p, len); -- cgit v1.2.3 From f8c36c58accd5c53a472b5c289910565b3df9f9d Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Wed, 11 Jul 2012 08:24:45 -0500 Subject: libceph: define ceph_extract_encoded_string() This adds a new utility routine which will return a dynamically- allocated buffer containing a string that has been decoded from ceph over-the-wire format. It also returns the length of the string if the address of a size variable is supplied to receive it. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/decode.h | 47 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/decode.h b/include/linux/ceph/decode.h index bcbd66c84890..4bbf2db45f46 100644 --- a/include/linux/ceph/decode.h +++ b/include/linux/ceph/decode.h @@ -1,6 +1,7 @@ #ifndef __CEPH_DECODE_H #define __CEPH_DECODE_H +#include #include #include #include @@ -84,6 +85,52 @@ static inline int ceph_has_room(void **p, void *end, size_t n) ceph_decode_copy(p, pv, n); \ } while (0) +/* + * Allocate a buffer big enough to hold the wire-encoded string, and + * decode the string into it. The resulting string will always be + * terminated with '\0'. If successful, *p will be advanced + * past the decoded data. Also, if lenp is not a null pointer, the + * length (not including the terminating '\0') will be recorded in + * *lenp. Note that a zero-length string is a valid return value. + * + * Returns a pointer to the newly-allocated string buffer, or a + * pointer-coded errno if an error occurs. Neither *p nor *lenp + * will have been updated if an error is returned. + * + * There are two possible failures: + * - converting the string would require accessing memory at or + * beyond the "end" pointer provided (-E + * - memory could not be allocated for the result + */ +static inline char *ceph_extract_encoded_string(void **p, void *end, + size_t *lenp, gfp_t gfp) +{ + u32 len; + void *sp = *p; + char *buf; + + ceph_decode_32_safe(&sp, end, len, bad); + if (!ceph_has_room(&sp, end, len)) + goto bad; + + buf = kmalloc(len + 1, gfp); + if (!buf) + return ERR_PTR(-ENOMEM); + + if (len) + memcpy(buf, sp, len); + buf[len] = '\0'; + + *p = (char *) *p + sizeof (u32) + len; + if (lenp) + *lenp = (size_t) len; + + return buf; + +bad: + return ERR_PTR(-ERANGE); +} + /* * struct ceph_timespec <-> struct timespec */ -- cgit v1.2.3 From 1fe60e51a3744528f3939b1b1167ca909133d9ae Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 30 Jul 2012 16:23:22 -0700 Subject: libceph: move feature bits to separate header This is simply cleanup that will keep things more closely synced with the userland code. Signed-off-by: Sage Weil Reviewed-by: Alex Elder Reviewed-by: Yehuda Sadeh --- include/linux/ceph/ceph_features.h | 24 ++++++++++++++++++++++++ include/linux/ceph/ceph_fs.h | 14 -------------- include/linux/ceph/libceph.h | 6 ------ 3 files changed, 24 insertions(+), 20 deletions(-) create mode 100644 include/linux/ceph/ceph_features.h (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/ceph_features.h b/include/linux/ceph/ceph_features.h new file mode 100644 index 000000000000..342f93dbe162 --- /dev/null +++ b/include/linux/ceph/ceph_features.h @@ -0,0 +1,24 @@ +#ifndef __CEPH_FEATURES +#define __CEPH_FEATURES + +/* + * feature bits + */ +#define CEPH_FEATURE_UID (1<<0) +#define CEPH_FEATURE_NOSRCADDR (1<<1) +#define CEPH_FEATURE_MONCLOCKCHECK (1<<2) +#define CEPH_FEATURE_FLOCK (1<<3) +#define CEPH_FEATURE_SUBSCRIBE2 (1<<4) +#define CEPH_FEATURE_MONNAMES (1<<5) +#define CEPH_FEATURE_RECONNECT_SEQ (1<<6) +#define CEPH_FEATURE_DIRLAYOUTHASH (1<<7) + +/* + * Features supported. + */ +#define CEPH_FEATURES_SUPPORTED_DEFAULT \ + (CEPH_FEATURE_NOSRCADDR) + +#define CEPH_FEATURES_REQUIRED_DEFAULT \ + (CEPH_FEATURE_NOSRCADDR) +#endif diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index e81ab30d4896..d021610efd65 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -35,20 +35,6 @@ /* arbitrary limit on max # of monitors (cluster of 3 is typical) */ #define CEPH_MAX_MON 31 - -/* - * feature bits - */ -#define CEPH_FEATURE_UID (1<<0) -#define CEPH_FEATURE_NOSRCADDR (1<<1) -#define CEPH_FEATURE_MONCLOCKCHECK (1<<2) -#define CEPH_FEATURE_FLOCK (1<<3) -#define CEPH_FEATURE_SUBSCRIBE2 (1<<4) -#define CEPH_FEATURE_MONNAMES (1<<5) -#define CEPH_FEATURE_RECONNECT_SEQ (1<<6) -#define CEPH_FEATURE_DIRLAYOUTHASH (1<<7) - - /* * ceph_file_layout - describe data layout for a file/inode */ diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index 98ec36ae8a3b..ea072e1f9db9 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -22,12 +22,6 @@ #include "osd_client.h" #include "ceph_fs.h" -/* - * Supported features - */ -#define CEPH_FEATURE_SUPPORTED_DEFAULT CEPH_FEATURE_NOSRCADDR -#define CEPH_FEATURE_REQUIRED_DEFAULT CEPH_FEATURE_NOSRCADDR - /* * mount options */ -- cgit v1.2.3 From 546f04ef716dd49521774653d8b032a7d64c05d9 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 30 Jul 2012 18:15:23 -0700 Subject: libceph: support crush tunables The server side recently added support for tuning some magic crush variables. Decode these variables if they are present, or use the default values if they are not present. Corresponds to ceph.git commit 89af369c25f274fe62ef730e5e8aad0c54f1e5a5. Signed-off-by: caleb miles Reviewed-by: Sage Weil Reviewed-by: Alex Elder Reviewed-by: Yehuda Sadeh --- include/linux/ceph/ceph_features.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/ceph_features.h b/include/linux/ceph/ceph_features.h index 342f93dbe162..dad579b0c0e6 100644 --- a/include/linux/ceph/ceph_features.h +++ b/include/linux/ceph/ceph_features.h @@ -12,12 +12,15 @@ #define CEPH_FEATURE_MONNAMES (1<<5) #define CEPH_FEATURE_RECONNECT_SEQ (1<<6) #define CEPH_FEATURE_DIRLAYOUTHASH (1<<7) +/* bits 8-17 defined by user-space; not supported yet here */ +#define CEPH_FEATURE_CRUSH_TUNABLES (1<<18) /* * Features supported. */ #define CEPH_FEATURES_SUPPORTED_DEFAULT \ - (CEPH_FEATURE_NOSRCADDR) + (CEPH_FEATURE_NOSRCADDR | \ + CEPH_FEATURE_CRUSH_TUNABLES) #define CEPH_FEATURES_REQUIRED_DEFAULT \ (CEPH_FEATURE_NOSRCADDR) -- cgit v1.2.3 From aa711ee3402ad10ffd5b70ce0417fadc9a95cccf Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Fri, 13 Jul 2012 20:35:11 -0500 Subject: ceph: define snap counts as u32 everywhere There are two structures in which a count of snapshots are maintained: struct ceph_snap_context { ... u32 num_snaps; ... } and struct ceph_snap_realm { ... u32 num_prior_parent_snaps; /* had prior to parent_since */ ... u32 num_snaps; ... } These fields never take on negative values (e.g., to hold special meaning), and so are really inherently unsigned. Furthermore they take their value from over-the-wire or on-disk formatted 32-bit values. So change their definition to have type u32, and change some spots elsewhere in the code to account for this change. Signed-off-by: Alex Elder Reviewed-by: Josh Durgin --- include/linux/ceph/libceph.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index ea072e1f9db9..42624789b06f 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -154,7 +154,7 @@ struct ceph_client { struct ceph_snap_context { atomic_t nref; u64 seq; - int num_snaps; + u32 num_snaps; u64 snaps[]; }; -- cgit v1.2.3 From 8dacc7da69a491c515851e68de6036f21b5663ce Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 20 Jul 2012 17:24:40 -0700 Subject: libceph: replace connection state bits with states Use a simple set of 6 enumerated values for the socket states (CON_STATE_*) and use those instead of the state bits. All of the con->state checks are now under the protection of the con mutex, so this is safe. It also simplifies many of the state checks because we can check for anything other than the expected state instead of various bits for races we can think of. This appears to hold up well to stress testing both with and without socket failure injection on the server side. Signed-off-by: Sage Weil --- include/linux/ceph/messenger.h | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index a310d7fe6e29..d9c2b8f5abde 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -116,18 +116,6 @@ struct ceph_msg_pos { #define SOCK_CLOSED 11 /* socket state changed to closed */ #define BACKOFF 15 -/* - * ceph_connection states - */ -#define CONNECTING 1 -#define NEGOTIATING 2 -#define CONNECTED 5 -#define STANDBY 8 /* no outgoing messages, socket closed. we keep - * the ceph_connection around to maintain shared - * state with the peer. */ -#define CLOSED 10 /* we've closed the connection */ -#define OPENING 13 /* open connection w/ (possibly new) peer */ - /* * A single connection with another host. * -- cgit v1.2.3 From 4a8616920860920abaa51193146fe36b38ef09aa Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 20 Jul 2012 17:29:55 -0700 Subject: libceph: clean up con flags Rename flags with CON_FLAG prefix, move the definitions into the c file, and (better) document their meaning. Signed-off-by: Sage Weil --- include/linux/ceph/messenger.h | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'include/linux/ceph') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index d9c2b8f5abde..189ae0637634 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -106,16 +106,6 @@ struct ceph_msg_pos { #define BASE_DELAY_INTERVAL (HZ/2) #define MAX_DELAY_INTERVAL (5 * 60 * HZ) -/* - * ceph_connection flag bits - */ - -#define LOSSYTX 0 /* we can close channel or drop messages on errors */ -#define KEEPALIVE_PENDING 3 -#define WRITE_PENDING 4 /* we have data ready to send */ -#define SOCK_CLOSED 11 /* socket state changed to closed */ -#define BACKOFF 15 - /* * A single connection with another host. * -- cgit v1.2.3