From f24e9980eb860d8600cbe5ef3d2fd9295320d229 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 6 Oct 2009 11:31:10 -0700 Subject: ceph: OSD client The OSD client is responsible for reading and writing data from/to the object storage pool. This includes determining where objects are stored in the cluster, and ensuring that requests are retried or redirected in the event of a node failure or data migration. If an OSD does not respond before a timeout expires, keepalive messages are sent across the lossless, ordered communications channel to ensure that any break in the TCP is discovered. If the session does reset, a reconnection is attempted and affected requests are resent (by the message transport layer). Signed-off-by: Sage Weil --- fs/ceph/osdmap.h | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 fs/ceph/osdmap.h (limited to 'fs/ceph/osdmap.h') diff --git a/fs/ceph/osdmap.h b/fs/ceph/osdmap.h new file mode 100644 index 000000000000..07127c6fb134 --- /dev/null +++ b/fs/ceph/osdmap.h @@ -0,0 +1,123 @@ +#ifndef _FS_CEPH_OSDMAP_H +#define _FS_CEPH_OSDMAP_H + +#include +#include "types.h" +#include "ceph_fs.h" +#include "crush/crush.h" + +/* + * The osd map describes the current membership of the osd cluster and + * specifies the mapping of objects to placement groups and placement + * groups to (sets of) osds. That is, it completely specifies the + * (desired) distribution of all data objects in the system at some + * point in time. + * + * Each map version is identified by an epoch, which increases monotonically. + * + * The map can be updated either via an incremental map (diff) describing + * the change between two successive epochs, or as a fully encoded map. + */ +struct ceph_pg_pool_info { + struct ceph_pg_pool v; + int pg_num_mask, pgp_num_mask, lpg_num_mask, lpgp_num_mask; +}; + +struct ceph_pg_mapping { + struct rb_node node; + u64 pgid; + int len; + int osds[]; +}; + +struct ceph_osdmap { + struct ceph_fsid fsid; + u32 epoch; + u32 mkfs_epoch; + struct ceph_timespec created, modified; + + u32 flags; /* CEPH_OSDMAP_* */ + + u32 max_osd; /* size of osd_state, _offload, _addr arrays */ + u8 *osd_state; /* CEPH_OSD_* */ + u32 *osd_weight; /* 0 = failed, 0x10000 = 100% normal */ + struct ceph_entity_addr *osd_addr; + + struct rb_root pg_temp; + + u32 num_pools; + struct ceph_pg_pool_info *pg_pool; + + /* the CRUSH map specifies the mapping of placement groups to + * the list of osds that store+replicate them. */ + struct crush_map *crush; +}; + +/* + * file layout helpers + */ +#define ceph_file_layout_su(l) ((__s32)le32_to_cpu((l).fl_stripe_unit)) +#define ceph_file_layout_stripe_count(l) \ + ((__s32)le32_to_cpu((l).fl_stripe_count)) +#define ceph_file_layout_object_size(l) ((__s32)le32_to_cpu((l).fl_object_size)) +#define ceph_file_layout_cas_hash(l) ((__s32)le32_to_cpu((l).fl_cas_hash)) +#define ceph_file_layout_object_su(l) \ + ((__s32)le32_to_cpu((l).fl_object_stripe_unit)) +#define ceph_file_layout_pg_preferred(l) \ + ((__s32)le32_to_cpu((l).fl_pg_preferred)) +#define ceph_file_layout_pg_pool(l) \ + ((__s32)le32_to_cpu((l).fl_pg_pool)) + +static inline unsigned ceph_file_layout_stripe_width(struct ceph_file_layout *l) +{ + return le32_to_cpu(l->fl_stripe_unit) * + le32_to_cpu(l->fl_stripe_count); +} + +/* "period" == bytes before i start on a new set of objects */ +static inline unsigned ceph_file_layout_period(struct ceph_file_layout *l) +{ + return le32_to_cpu(l->fl_object_size) * + le32_to_cpu(l->fl_stripe_count); +} + + +static inline int ceph_osd_is_up(struct ceph_osdmap *map, int osd) +{ + return (osd < map->max_osd) && (map->osd_state[osd] & CEPH_OSD_UP); +} + +static inline bool ceph_osdmap_flag(struct ceph_osdmap *map, int flag) +{ + return map && (map->flags & flag); +} + +extern char *ceph_osdmap_state_str(char *str, int len, int state); + +static inline struct ceph_entity_addr *ceph_osd_addr(struct ceph_osdmap *map, + int osd) +{ + if (osd >= map->max_osd) + return NULL; + return &map->osd_addr[osd]; +} + +extern struct ceph_osdmap *osdmap_decode(void **p, void *end); +extern struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end, + struct ceph_osdmap *map, + struct ceph_messenger *msgr); +extern void ceph_osdmap_destroy(struct ceph_osdmap *map); + +/* calculate mapping of a file extent to an object */ +extern void ceph_calc_file_object_mapping(struct ceph_file_layout *layout, + u64 off, u64 *plen, + u64 *bno, u64 *oxoff, u64 *oxlen); + +/* calculate mapping of object to a placement group */ +extern int ceph_calc_object_layout(struct ceph_object_layout *ol, + const char *oid, + struct ceph_file_layout *fl, + struct ceph_osdmap *osdmap); +extern int ceph_calc_pg_primary(struct ceph_osdmap *osdmap, union ceph_pg pgid); + +#endif -- cgit v1.2.3 From 51042122d4f85e0f8ee577a4230f172fcc57c456 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 4 Nov 2009 11:39:12 -0800 Subject: ceph: fix endian conversions for ceph_pg The endian conversions don't quite work with the old union ceph_pg. Just make it a regular struct, and make each field __le. This is simpler and it has the added bonus of actually working. Signed-off-by: Sage Weil --- fs/ceph/ioctl.c | 4 +-- fs/ceph/osd_client.c | 8 ++--- fs/ceph/osdmap.c | 100 +++++++++++++++++++++++++++++++-------------------- fs/ceph/osdmap.h | 5 +-- fs/ceph/rados.h | 13 +++---- 5 files changed, 75 insertions(+), 55 deletions(-) (limited to 'fs/ceph/osdmap.h') diff --git a/fs/ceph/ioctl.c b/fs/ceph/ioctl.c index e4f99eff5d93..4c33e19fc241 100644 --- a/fs/ceph/ioctl.c +++ b/fs/ceph/ioctl.c @@ -99,7 +99,7 @@ static long ceph_ioctl_get_dataloc(struct file *file, void __user *arg) u64 len = 1, olen; u64 tmp; struct ceph_object_layout ol; - union ceph_pg pgid; + struct ceph_pg pgid; /* copy and validate */ if (copy_from_user(&dl, arg, sizeof(dl))) @@ -121,7 +121,7 @@ static long ceph_ioctl_get_dataloc(struct file *file, void __user *arg) ceph_calc_object_layout(&ol, dl.object_name, &ci->i_layout, osdc->osdmap); - pgid.pg64 = le64_to_cpu(ol.ol_pgid); + pgid = ol.ol_pgid; dl.osd = ceph_calc_pg_primary(osdc->osdmap, pgid); if (dl.osd >= 0) { struct ceph_entity_addr *a = diff --git a/fs/ceph/osd_client.c b/fs/ceph/osd_client.c index 7dc0f6299a52..7db14ba6261c 100644 --- a/fs/ceph/osd_client.c +++ b/fs/ceph/osd_client.c @@ -520,7 +520,7 @@ static int __map_osds(struct ceph_osd_client *osdc, struct ceph_osd_request *req) { struct ceph_osd_request_head *reqhead = req->r_request->front.iov_base; - union ceph_pg pgid; + struct ceph_pg pgid; int o = -1; int err; struct ceph_osd *newosd = NULL; @@ -530,7 +530,7 @@ static int __map_osds(struct ceph_osd_client *osdc, &req->r_file_layout, osdc->osdmap); if (err) return err; - pgid.pg64 = le64_to_cpu(reqhead->layout.ol_pgid); + pgid = reqhead->layout.ol_pgid; o = ceph_calc_pg_primary(osdc->osdmap, pgid); if ((req->r_osd && req->r_osd->o_osd == o && @@ -538,8 +538,8 @@ static int __map_osds(struct ceph_osd_client *osdc, (req->r_osd == NULL && o == -1)) return 0; /* no change */ - dout("map_osds tid %llu pgid %llx pool %d osd%d (was osd%d)\n", - req->r_tid, pgid.pg64, pgid.pg.pool, o, + dout("map_osds tid %llu pgid %d.%x osd%d (was osd%d)\n", + req->r_tid, le32_to_cpu(pgid.pool), le16_to_cpu(pgid.ps), o, req->r_osd ? req->r_osd->o_osd : -1); if (req->r_osd) { diff --git a/fs/ceph/osdmap.c b/fs/ceph/osdmap.c index cd7bb265d789..8b0cd1107507 100644 --- a/fs/ceph/osdmap.c +++ b/fs/ceph/osdmap.c @@ -366,19 +366,33 @@ static int osdmap_set_max_osd(struct ceph_osdmap *map, int max) /* * Insert a new pg_temp mapping */ +static int pgid_cmp(struct ceph_pg l, struct ceph_pg r) +{ + u64 a = *(u64 *)&l; + u64 b = *(u64 *)&r; + + if (a < b) + return -1; + if (a > b) + return 1; + return 0; +} + static int __insert_pg_mapping(struct ceph_pg_mapping *new, struct rb_root *root) { struct rb_node **p = &root->rb_node; struct rb_node *parent = NULL; struct ceph_pg_mapping *pg = NULL; + int c; while (*p) { parent = *p; pg = rb_entry(parent, struct ceph_pg_mapping, node); - if (new->pgid < pg->pgid) + c = pgid_cmp(new->pgid, pg->pgid); + if (c < 0) p = &(*p)->rb_left; - else if (new->pgid > pg->pgid) + else if (c > 0) p = &(*p)->rb_right; else return -EEXIST; @@ -467,11 +481,11 @@ struct ceph_osdmap *osdmap_decode(void **p, void *end) ceph_decode_32_safe(p, end, len, bad); for (i = 0; i < len; i++) { int n, j; - u64 pgid; + struct ceph_pg pgid; struct ceph_pg_mapping *pg; ceph_decode_need(p, end, sizeof(u32) + sizeof(u64), bad); - pgid = ceph_decode_64(p); + ceph_decode_copy(p, &pgid, sizeof(pgid)); n = ceph_decode_32(p); ceph_decode_need(p, end, n * sizeof(u32), bad); pg = kmalloc(sizeof(*pg) + n*sizeof(u32), GFP_NOFS); @@ -487,7 +501,7 @@ struct ceph_osdmap *osdmap_decode(void **p, void *end) err = __insert_pg_mapping(pg, &map->pg_temp); if (err) goto bad; - dout(" added pg_temp %llx len %d\n", pgid, len); + dout(" added pg_temp %llx len %d\n", *(u64 *)&pgid, len); } /* crush */ @@ -659,19 +673,20 @@ struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end, while (len--) { struct ceph_pg_mapping *pg; int j; - u64 pgid; + struct ceph_pg pgid; u32 pglen; ceph_decode_need(p, end, sizeof(u64) + sizeof(u32), bad); - pgid = ceph_decode_64(p); + ceph_decode_copy(p, &pgid, sizeof(pgid)); pglen = ceph_decode_32(p); /* remove any? */ - while (rbp && rb_entry(rbp, struct ceph_pg_mapping, - node)->pgid <= pgid) { + while (rbp && pgid_cmp(rb_entry(rbp, struct ceph_pg_mapping, + node)->pgid, pgid) <= 0) { struct rb_node *cur = rbp; rbp = rb_next(rbp); dout(" removed pg_temp %llx\n", - rb_entry(cur, struct ceph_pg_mapping, node)->pgid); + *(u64 *)&rb_entry(cur, struct ceph_pg_mapping, + node)->pgid); rb_erase(cur, &map->pg_temp); } @@ -690,14 +705,16 @@ struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end, err = __insert_pg_mapping(pg, &map->pg_temp); if (err) goto bad; - dout(" added pg_temp %llx len %d\n", pgid, pglen); + dout(" added pg_temp %llx len %d\n", *(u64 *)&pgid, + pglen); } } while (rbp) { struct rb_node *cur = rbp; rbp = rb_next(rbp); dout(" removed pg_temp %llx\n", - rb_entry(cur, struct ceph_pg_mapping, node)->pgid); + *(u64 *)&rb_entry(cur, struct ceph_pg_mapping, + node)->pgid); rb_erase(cur, &map->pg_temp); } @@ -782,16 +799,19 @@ int ceph_calc_object_layout(struct ceph_object_layout *ol, struct ceph_osdmap *osdmap) { unsigned num, num_mask; - union ceph_pg pgid; + struct ceph_pg pgid; s32 preferred = (s32)le32_to_cpu(fl->fl_pg_preferred); int poolid = le32_to_cpu(fl->fl_pg_pool); struct ceph_pg_pool_info *pool; + unsigned ps; if (poolid >= osdmap->num_pools) return -EIO; - pool = &osdmap->pg_pool[poolid]; + pool = &osdmap->pg_pool[poolid]; + ps = ceph_full_name_hash(oid, strlen(oid)); if (preferred >= 0) { + ps += preferred; num = le32_to_cpu(pool->v.lpg_num); num_mask = pool->lpg_num_mask; } else { @@ -799,22 +819,17 @@ int ceph_calc_object_layout(struct ceph_object_layout *ol, num_mask = pool->pg_num_mask; } - pgid.pg64 = 0; /* start with it zeroed out */ - pgid.pg.ps = ceph_full_name_hash(oid, strlen(oid)); - pgid.pg.preferred = preferred; - if (preferred >= 0) - pgid.pg.ps += preferred; - pgid.pg.pool = le32_to_cpu(fl->fl_pg_pool); + pgid.ps = cpu_to_le16(ps); + pgid.preferred = cpu_to_le16(preferred); + pgid.pool = fl->fl_pg_pool; if (preferred >= 0) - dout("calc_object_layout '%s' pgid %d.%xp%d (%llx)\n", oid, - pgid.pg.pool, pgid.pg.ps, (int)preferred, pgid.pg64); + dout("calc_object_layout '%s' pgid %d.%xp%d\n", oid, poolid, ps, + (int)preferred); else - dout("calc_object_layout '%s' pgid %d.%x (%llx)\n", oid, - pgid.pg.pool, pgid.pg.ps, pgid.pg64); + dout("calc_object_layout '%s' pgid %d.%x\n", oid, poolid, ps); - ol->ol_pgid = cpu_to_le64(pgid.pg64); + ol->ol_pgid = pgid; ol->ol_stripe_unit = fl->fl_object_stripe_unit; - return 0; } @@ -822,21 +837,24 @@ int ceph_calc_object_layout(struct ceph_object_layout *ol, * Calculate raw osd vector for the given pgid. Return pointer to osd * array, or NULL on failure. */ -static int *calc_pg_raw(struct ceph_osdmap *osdmap, union ceph_pg pgid, +static int *calc_pg_raw(struct ceph_osdmap *osdmap, struct ceph_pg pgid, int *osds, int *num) { struct rb_node *n = osdmap->pg_temp.rb_node; struct ceph_pg_mapping *pg; struct ceph_pg_pool_info *pool; int ruleno; - unsigned pps; /* placement ps */ + unsigned poolid, ps, pps; + int preferred; + int c; /* pg_temp? */ while (n) { pg = rb_entry(n, struct ceph_pg_mapping, node); - if (pgid.pg64 < pg->pgid) + c = pgid_cmp(pgid, pg->pgid); + if (c < 0) n = n->rb_left; - else if (pgid.pg64 > pg->pgid) + else if (c > 0) n = n->rb_right; else { *num = pg->len; @@ -845,36 +863,40 @@ static int *calc_pg_raw(struct ceph_osdmap *osdmap, union ceph_pg pgid, } /* crush */ - if (pgid.pg.pool >= osdmap->num_pools) + poolid = le32_to_cpu(pgid.pool); + ps = le16_to_cpu(pgid.ps); + preferred = (s16)le16_to_cpu(pgid.preferred); + + if (poolid >= osdmap->num_pools) return NULL; - pool = &osdmap->pg_pool[pgid.pg.pool]; + pool = &osdmap->pg_pool[poolid]; ruleno = crush_find_rule(osdmap->crush, pool->v.crush_ruleset, pool->v.type, pool->v.size); if (ruleno < 0) { pr_err("no crush rule pool %d type %d size %d\n", - pgid.pg.pool, pool->v.type, pool->v.size); + poolid, pool->v.type, pool->v.size); return NULL; } - if (pgid.pg.preferred >= 0) - pps = ceph_stable_mod(pgid.pg.ps, + if (preferred >= 0) + pps = ceph_stable_mod(ps, le32_to_cpu(pool->v.lpgp_num), pool->lpgp_num_mask); else - pps = ceph_stable_mod(pgid.pg.ps, + pps = ceph_stable_mod(ps, le32_to_cpu(pool->v.pgp_num), pool->pgp_num_mask); - pps += pgid.pg.pool; + pps += poolid; *num = crush_do_rule(osdmap->crush, ruleno, pps, osds, min_t(int, pool->v.size, *num), - pgid.pg.preferred, osdmap->osd_weight); + preferred, osdmap->osd_weight); return osds; } /* * Return primary osd for given pgid, or -1 if none. */ -int ceph_calc_pg_primary(struct ceph_osdmap *osdmap, union ceph_pg pgid) +int ceph_calc_pg_primary(struct ceph_osdmap *osdmap, struct ceph_pg pgid) { int rawosds[10], *osds; int i, num = ARRAY_SIZE(rawosds); diff --git a/fs/ceph/osdmap.h b/fs/ceph/osdmap.h index 07127c6fb134..c4af8418aa00 100644 --- a/fs/ceph/osdmap.h +++ b/fs/ceph/osdmap.h @@ -25,7 +25,7 @@ struct ceph_pg_pool_info { struct ceph_pg_mapping { struct rb_node node; - u64 pgid; + struct ceph_pg pgid; int len; int osds[]; }; @@ -118,6 +118,7 @@ extern int ceph_calc_object_layout(struct ceph_object_layout *ol, const char *oid, struct ceph_file_layout *fl, struct ceph_osdmap *osdmap); -extern int ceph_calc_pg_primary(struct ceph_osdmap *osdmap, union ceph_pg pgid); +extern int ceph_calc_pg_primary(struct ceph_osdmap *osdmap, + struct ceph_pg pgid); #endif diff --git a/fs/ceph/rados.h b/fs/ceph/rados.h index a48cf4ae391e..85bdef78d142 100644 --- a/fs/ceph/rados.h +++ b/fs/ceph/rados.h @@ -55,13 +55,10 @@ struct ceph_timespec { * placement group. * we encode this into one __le64. */ -union ceph_pg { - __u64 pg64; - struct { - __s16 preferred; /* preferred primary osd */ - __u16 ps; /* placement seed */ - __u32 pool; /* object pool */ - } __attribute__ ((packed)) pg; +struct ceph_pg { + __le16 preferred; /* preferred primary osd */ + __le16 ps; /* placement seed */ + __le32 pool; /* object pool */ } __attribute__ ((packed)); /* @@ -117,7 +114,7 @@ static inline int ceph_stable_mod(int x, int b, int bmask) * object layout - how a given object should be stored. */ struct ceph_object_layout { - __le64 ol_pgid; /* raw pg, with _full_ ps precision. */ + struct ceph_pg ol_pgid; /* raw pg, with _full_ ps precision. */ __le32 ol_stripe_unit; /* for per-object parity, if any */ } __attribute__ ((packed)); -- cgit v1.2.3 From 4fc51be8fa7043ff9a1e34fef0e99214373332ac Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 16 Feb 2010 15:55:03 -0800 Subject: ceph: use rbtree for pg pools; decode new osdmap format Since we can now create and destroy pg pools, the pool ids will be sparse, and an array no longer makes sense for looking up by pool id. Use an rbtree instead. The OSDMap encoding also no longer has a max pool count (previously used to allocate the array). There is a new pool_max, that is the largest pool id we've ever used, although we don't actually need it in the client. Signed-off-by: Sage Weil --- fs/ceph/debugfs.c | 7 +-- fs/ceph/osdmap.c | 136 +++++++++++++++++++++++++++++++++++++----------------- fs/ceph/osdmap.h | 7 +-- fs/ceph/rados.h | 4 +- 4 files changed, 104 insertions(+), 50 deletions(-) (limited to 'fs/ceph/osdmap.h') diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c index 1a47b5c25b5f..e159f1415110 100644 --- a/fs/ceph/debugfs.c +++ b/fs/ceph/debugfs.c @@ -78,6 +78,7 @@ static int osdmap_show(struct seq_file *s, void *p) { int i; struct ceph_client *client = s->private; + struct rb_node *n; if (client->osdc.osdmap == NULL) return 0; @@ -87,11 +88,11 @@ static int osdmap_show(struct seq_file *s, void *p) " NEARFULL" : "", (client->osdc.osdmap->flags & CEPH_OSDMAP_FULL) ? " FULL" : ""); - for (i = 0; i < client->osdc.osdmap->num_pools; i++) { + for (n = rb_first(&client->osdc.osdmap->pg_pools); n; n = rb_next(n)) { struct ceph_pg_pool_info *pool = - &client->osdc.osdmap->pg_pool[i]; + rb_entry(n, struct ceph_pg_pool_info, node); seq_printf(s, "pg_pool %d pg_num %d / %d, lpg_num %d / %d\n", - i, pool->v.pg_num, pool->pg_num_mask, + pool->id, pool->v.pg_num, pool->pg_num_mask, pool->v.lpg_num, pool->lpg_num_mask); } for (i = 0; i < client->osdc.osdmap->max_osd; i++) { diff --git a/fs/ceph/osdmap.c b/fs/ceph/osdmap.c index 443fdcdb19c4..34b5696c84fd 100644 --- a/fs/ceph/osdmap.c +++ b/fs/ceph/osdmap.c @@ -328,9 +328,15 @@ void ceph_osdmap_destroy(struct ceph_osdmap *map) rb_erase(&pg->node, &map->pg_temp); kfree(pg); } + while (!RB_EMPTY_ROOT(&map->pg_pools)) { + struct ceph_pg_pool_info *pi = + rb_entry(rb_first(&map->pg_pools), + struct ceph_pg_pool_info, node); + rb_erase(&pi->node, &map->pg_pools); + kfree(pi); + } kfree(map->osd_state); kfree(map->osd_weight); - kfree(map->pg_pool); kfree(map->osd_addr); kfree(map); } @@ -432,6 +438,48 @@ static struct ceph_pg_mapping *__lookup_pg_mapping(struct rb_root *root, return NULL; } +/* + * rbtree of pg pool info + */ +static int __insert_pg_pool(struct rb_root *root, struct ceph_pg_pool_info *new) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct ceph_pg_pool_info *pi = NULL; + + while (*p) { + parent = *p; + pi = rb_entry(parent, struct ceph_pg_pool_info, node); + if (new->id < pi->id) + p = &(*p)->rb_left; + else if (new->id > pi->id) + p = &(*p)->rb_right; + else + return -EEXIST; + } + + rb_link_node(&new->node, parent, p); + rb_insert_color(&new->node, root); + return 0; +} + +static struct ceph_pg_pool_info *__lookup_pg_pool(struct rb_root *root, int id) +{ + struct ceph_pg_pool_info *pi; + struct rb_node *n = root->rb_node; + + while (n) { + pi = rb_entry(n, struct ceph_pg_pool_info, node); + if (id < pi->id) + n = n->rb_left; + else if (id > pi->id) + n = n->rb_right; + else + return pi; + } + return NULL; +} + /* * decode a full map. */ @@ -443,6 +491,7 @@ struct ceph_osdmap *osdmap_decode(void **p, void *end) u8 ev; int err = -EINVAL; void *start = *p; + struct ceph_pg_pool_info *pi; dout("osdmap_decode %p to %p len %d\n", *p, end, (int)(end - *p)); @@ -464,32 +513,27 @@ struct ceph_osdmap *osdmap_decode(void **p, void *end) ceph_decode_copy(p, &map->created, sizeof(map->created)); ceph_decode_copy(p, &map->modified, sizeof(map->modified)); - map->num_pools = ceph_decode_32(p); - map->pg_pool = kcalloc(map->num_pools, sizeof(*map->pg_pool), - GFP_NOFS); - if (!map->pg_pool) { - err = -ENOMEM; - goto bad; - } ceph_decode_32_safe(p, end, max, bad); while (max--) { - ceph_decode_need(p, end, 4+1+sizeof(map->pg_pool->v), bad); - i = ceph_decode_32(p); - if (i >= map->num_pools) + ceph_decode_need(p, end, 4 + 1 + sizeof(pi->v), bad); + pi = kmalloc(sizeof(*pi), GFP_NOFS); + if (!pi) goto bad; + pi->id = ceph_decode_32(p); ev = ceph_decode_8(p); /* encoding version */ if (ev > CEPH_PG_POOL_VERSION) { pr_warning("got unknown v %d > %d of ceph_pg_pool\n", ev, CEPH_PG_POOL_VERSION); goto bad; } - ceph_decode_copy(p, &map->pg_pool[i].v, - sizeof(map->pg_pool->v)); - calc_pg_masks(&map->pg_pool[i]); - p += le32_to_cpu(map->pg_pool[i].v.num_snaps) * sizeof(u64); - p += le32_to_cpu(map->pg_pool[i].v.num_removed_snap_intervals) + ceph_decode_copy(p, &pi->v, sizeof(pi->v)); + __insert_pg_pool(&map->pg_pools, pi); + calc_pg_masks(pi); + p += le32_to_cpu(pi->v.num_snaps) * sizeof(u64); + p += le32_to_cpu(pi->v.num_removed_snap_intervals) * sizeof(u64) * 2; } + ceph_decode_32_safe(p, end, map->pool_max, bad); ceph_decode_32_safe(p, end, map->flags, bad); @@ -581,7 +625,7 @@ struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end, u32 epoch = 0; struct ceph_timespec modified; u32 len, pool; - __s32 new_flags, max; + __s32 new_pool_max, new_flags, max; void *start = *p; int err = -EINVAL; u16 version; @@ -600,6 +644,7 @@ struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end, epoch = ceph_decode_32(p); BUG_ON(epoch != map->epoch+1); ceph_decode_copy(p, &modified, sizeof(modified)); + new_pool_max = ceph_decode_32(p); new_flags = ceph_decode_32(p); /* full map? */ @@ -623,6 +668,8 @@ struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end, /* new flags? */ if (new_flags >= 0) map->flags = new_flags; + if (new_pool_max >= 0) + map->pool_max = new_pool_max; ceph_decode_need(p, end, 5*sizeof(u32), bad); @@ -647,37 +694,42 @@ struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end, ceph_decode_32_safe(p, end, len, bad); while (len--) { __u8 ev; + struct ceph_pg_pool_info *pi; ceph_decode_32_safe(p, end, pool, bad); - if (pool >= map->num_pools) { - void *pg_pool = kcalloc(pool + 1, - sizeof(*map->pg_pool), - GFP_NOFS); - if (!pg_pool) { - err = -ENOMEM; - goto bad; - } - memcpy(pg_pool, map->pg_pool, - map->num_pools * sizeof(*map->pg_pool)); - kfree(map->pg_pool); - map->pg_pool = pg_pool; - map->num_pools = pool+1; - } - ceph_decode_need(p, end, 1 + sizeof(map->pg_pool->v), bad); + ceph_decode_need(p, end, 1 + sizeof(pi->v), bad); ev = ceph_decode_8(p); /* encoding version */ if (ev > CEPH_PG_POOL_VERSION) { pr_warning("got unknown v %d > %d of ceph_pg_pool\n", ev, CEPH_PG_POOL_VERSION); goto bad; } - ceph_decode_copy(p, &map->pg_pool[pool].v, - sizeof(map->pg_pool->v)); - calc_pg_masks(&map->pg_pool[pool]); + pi = __lookup_pg_pool(&map->pg_pools, pool); + if (!pi) { + pi = kmalloc(sizeof(*pi), GFP_NOFS); + if (!pi) { + err = -ENOMEM; + goto bad; + } + pi->id = pool; + __insert_pg_pool(&map->pg_pools, pi); + } + ceph_decode_copy(p, &pi->v, sizeof(pi->v)); + calc_pg_masks(pi); } - /* old_pool (ignore) */ + /* old_pool */ ceph_decode_32_safe(p, end, len, bad); - *p += len * sizeof(u32); + while (len--) { + struct ceph_pg_pool_info *pi; + + ceph_decode_32_safe(p, end, pool, bad); + pi = __lookup_pg_pool(&map->pg_pools, pool); + if (pi) { + rb_erase(&pi->node, &map->pg_pools); + kfree(pi); + } + } /* new_up */ err = -EINVAL; @@ -861,10 +913,10 @@ int ceph_calc_object_layout(struct ceph_object_layout *ol, unsigned ps; BUG_ON(!osdmap); - if (poolid >= osdmap->num_pools) - return -EIO; - pool = &osdmap->pg_pool[poolid]; + pool = __lookup_pg_pool(&osdmap->pg_pools, poolid); + if (!pool) + return -EIO; ps = ceph_str_hash(pool->v.object_hash, oid, strlen(oid)); if (preferred >= 0) { ps += preferred; @@ -919,9 +971,9 @@ static int *calc_pg_raw(struct ceph_osdmap *osdmap, struct ceph_pg pgid, preferred >= osdmap->crush->max_devices) preferred = -1; - if (poolid >= osdmap->num_pools) + pool = __lookup_pg_pool(&osdmap->pg_pools, poolid); + if (!pool) return NULL; - pool = &osdmap->pg_pool[poolid]; ruleno = crush_find_rule(osdmap->crush, pool->v.crush_ruleset, pool->v.type, pool->v.size); if (ruleno < 0) { diff --git a/fs/ceph/osdmap.h b/fs/ceph/osdmap.h index c4af8418aa00..1fb55afb2642 100644 --- a/fs/ceph/osdmap.h +++ b/fs/ceph/osdmap.h @@ -19,6 +19,8 @@ * the change between two successive epochs, or as a fully encoded map. */ struct ceph_pg_pool_info { + struct rb_node node; + int id; struct ceph_pg_pool v; int pg_num_mask, pgp_num_mask, lpg_num_mask, lpgp_num_mask; }; @@ -44,9 +46,8 @@ struct ceph_osdmap { struct ceph_entity_addr *osd_addr; struct rb_root pg_temp; - - u32 num_pools; - struct ceph_pg_pool_info *pg_pool; + struct rb_root pg_pools; + u32 pool_max; /* the CRUSH map specifies the mapping of placement groups to * the list of osds that store+replicate them. */ diff --git a/fs/ceph/rados.h b/fs/ceph/rados.h index 1f4c78640541..26ac8b89a676 100644 --- a/fs/ceph/rados.h +++ b/fs/ceph/rados.h @@ -11,8 +11,8 @@ /* * osdmap encoding versions */ -#define CEPH_OSDMAP_INC_VERSION 3 -#define CEPH_OSDMAP_VERSION 3 +#define CEPH_OSDMAP_INC_VERSION 4 +#define CEPH_OSDMAP_VERSION 4 /* * fs id -- cgit v1.2.3