kernel/bpf/btf.c

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /* Copyright (c) 2018 Facebook */
   3
   4 #include <uapi/linux/btf.h>
   5 #include <uapi/linux/types.h>
   6 #include <linux/seq_file.h>
   7 #include <linux/compiler.h>
   8 #include <linux/ctype.h>
   9 #include <linux/errno.h>
  10 #include <linux/slab.h>
  11 #include <linux/anon_inodes.h>
  12 #include <linux/file.h>
  13 #include <linux/uaccess.h>
  14 #include <linux/kernel.h>
  15 #include <linux/idr.h>
  16 #include <linux/sort.h>
  17 #include <linux/bpf_verifier.h>
  18 #include <linux/btf.h>
  19
  20 /* BTF (BPF Type Format) is the meta data format which describes
  21  * the data types of BPF program/map.  Hence, it basically focus
  22  * on the C programming language which the modern BPF is primary
  23  * using.
  24  *
  25  * ELF Section:
  26  * ~~~~~~~~~~~
  27  * The BTF data is stored under the ".BTF" ELF section
  28  *
  29  * struct btf_type:
  30  * ~~~~~~~~~~~~~~~
  31  * Each 'struct btf_type' object describes a C data type.
  32  * Depending on the type it is describing, a 'struct btf_type'
  33  * object may be followed by more data.  F.e.
  34  * To describe an array, 'struct btf_type' is followed by
  35  * 'struct btf_array'.
  36  *
  37  * 'struct btf_type' and any extra data following it are
  38  * 4 bytes aligned.
  39  *
  40  * Type section:
  41  * ~~~~~~~~~~~~~
  42  * The BTF type section contains a list of 'struct btf_type' objects.
  43  * Each one describes a C type.  Recall from the above section
  44  * that a 'struct btf_type' object could be immediately followed by extra
  45  * data in order to desribe some particular C types.
  46  *
  47  * type_id:
  48  * ~~~~~~~
  49  * Each btf_type object is identified by a type_id.  The type_id
  50  * is implicitly implied by the location of the btf_type object in
  51  * the BTF type section.  The first one has type_id 1.  The second
  52  * one has type_id 2...etc.  Hence, an earlier btf_type has
  53  * a smaller type_id.
  54  *
  55  * A btf_type object may refer to another btf_type object by using
  56  * type_id (i.e. the "type" in the "struct btf_type").
  57  *
  58  * NOTE that we cannot assume any reference-order.
  59  * A btf_type object can refer to an earlier btf_type object
  60  * but it can also refer to a later btf_type object.
  61  *
  62  * For example, to describe "const void *".  A btf_type
  63  * object describing "const" may refer to another btf_type
  64  * object describing "void *".  This type-reference is done
  65  * by specifying type_id:
  66  *
  67  * [1] CONST (anon) type_id=2
  68  * [2] PTR (anon) type_id=0
  69  *
  70  * The above is the btf_verifier debug log:
  71  *   - Each line started with "[?]" is a btf_type object
  72  *   - [?] is the type_id of the btf_type object.
  73  *   - CONST/PTR is the BTF_KIND_XXX
  74  *   - "(anon)" is the name of the type.  It just
  75  *     happens that CONST and PTR has no name.
  76  *   - type_id=XXX is the 'u32 type' in btf_type
  77  *
  78  * NOTE: "void" has type_id 0
  79  *
  80  * String section:
  81  * ~~~~~~~~~~~~~~
  82  * The BTF string section contains the names used by the type section.
  83  * Each string is referred by an "offset" from the beginning of the
  84  * string section.
  85  *
  86  * Each string is '\0' terminated.
  87  *
  88  * The first character in the string section must be '\0'
  89  * which is used to mean 'anonymous'. Some btf_type may not
  90  * have a name.
  91  */
  92
  93 /* BTF verification:
  94  *
  95  * To verify BTF data, two passes are needed.
  96  *
  97  * Pass #1
  98  * ~~~~~~~
  99  * The first pass is to collect all btf_type objects to
 100  * an array: "btf->types".
 101  *
 102  * Depending on the C type that a btf_type is describing,
 103  * a btf_type may be followed by extra data.  We don't know
 104  * how many btf_type is there, and more importantly we don't
 105  * know where each btf_type is located in the type section.
 106  *
 107  * Without knowing the location of each type_id, most verifications
 108  * cannot be done.  e.g. an earlier btf_type may refer to a later
 109  * btf_type (recall the "const void *" above), so we cannot
 110  * check this type-reference in the first pass.
 111  *
 112  * In the first pass, it still does some verifications (e.g.
 113  * checking the name is a valid offset to the string section).
 114  *
 115  * Pass #2
 116  * ~~~~~~~
 117  * The main focus is to resolve a btf_type that is referring
 118  * to another type.
 119  *
 120  * We have to ensure the referring type:
 121  * 1) does exist in the BTF (i.e. in btf->types[])
 122  * 2) does not cause a loop:
 123  *      struct A {
 124  *              struct B b;
 125  *      };
 126  *
 127  *      struct B {
 128  *              struct A a;
 129  *      };
 130  *
 131  * btf_type_needs_resolve() decides if a btf_type needs
 132  * to be resolved.
 133  *
 134  * The needs_resolve type implements the "resolve()" ops which
 135  * essentially does a DFS and detects backedge.
 136  *
 137  * During resolve (or DFS), different C types have different
 138  * "RESOLVED" conditions.
 139  *
 140  * When resolving a BTF_KIND_STRUCT, we need to resolve all its
 141  * members because a member is always referring to another
 142  * type.  A struct's member can be treated as "RESOLVED" if
 143  * it is referring to a BTF_KIND_PTR.  Otherwise, the
 144  * following valid C struct would be rejected:
 145  *
 146  *      struct A {
 147  *              int m;
 148  *              struct A *a;
 149  *      };
 150  *
 151  * When resolving a BTF_KIND_PTR, it needs to keep resolving if
 152  * it is referring to another BTF_KIND_PTR.  Otherwise, we cannot
 153  * detect a pointer loop, e.g.:
 154  * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR +
 155  *                        ^                                         |
 156  *                        +-----------------------------------------+
 157  *
 158  */
 159
 160 #define BITS_PER_U64 (sizeof(u64) * BITS_PER_BYTE)
 161 #define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
 162 #define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK)
 163 #define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3)
 164 #define BITS_ROUNDUP_BYTES(bits) \
 165         (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))
 166
 167 #define BTF_INFO_MASK 0x8f00ffff
 168 #define BTF_INT_MASK 0x0fffffff
 169 #define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE)
 170 #define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET)
 171
 172 /* 16MB for 64k structs and each has 16 members and
 173  * a few MB spaces for the string section.
 174  * The hard limit is S32_MAX.
 175  */
 176 #define BTF_MAX_SIZE (16 * 1024 * 1024)
 177
 178 #define for_each_member(i, struct_type, member)                 \
 179         for (i = 0, member = btf_type_member(struct_type);      \
 180              i < btf_type_vlen(struct_type);                    \
 181              i++, member++)
 182
 183 #define for_each_member_from(i, from, struct_type, member)              \
 184         for (i = from, member = btf_type_member(struct_type) + from;    \
 185              i < btf_type_vlen(struct_type);                            \
 186              i++, member++)
 187
 188 static DEFINE_IDR(btf_idr);
 189 static DEFINE_SPINLOCK(btf_idr_lock);
 190
 191 struct btf {
 192         void *data;
 193         struct btf_type **types;
 194         u32 *resolved_ids;
 195         u32 *resolved_sizes;
 196         const char *strings;
 197         void *nohdr_data;
 198         struct btf_header hdr;
 199         u32 nr_types;
 200         u32 types_size;
 201         u32 data_size;
 202         refcount_t refcnt;
 203         u32 id;
 204         struct rcu_head rcu;
 205 };
 206
 207 enum verifier_phase {
 208         CHECK_META,
 209         CHECK_TYPE,
 210 };
 211
 212 struct resolve_vertex {
 213         const struct btf_type *t;
 214         u32 type_id;
 215         u16 next_member;
 216 };
 217
 218 enum visit_state {
 219         NOT_VISITED,
 220         VISITED,
 221         RESOLVED,
 222 };
 223
 224 enum resolve_mode {
 225         RESOLVE_TBD,    /* To Be Determined */
 226         RESOLVE_PTR,    /* Resolving for Pointer */
 227         RESOLVE_STRUCT_OR_ARRAY,        /* Resolving for struct/union
 228                                          * or array
 229                                          */
 230 };
 231
 232 #define MAX_RESOLVE_DEPTH 32
 233
 234 struct btf_sec_info {
 235         u32 off;
 236         u32 len;
 237 };
 238
 239 struct btf_verifier_env {
 240         struct btf *btf;
 241         u8 *visit_states;
 242         struct resolve_vertex stack[MAX_RESOLVE_DEPTH];
 243         struct bpf_verifier_log log;
 244         u32 log_type_id;
 245         u32 top_stack;
 246         enum verifier_phase phase;
 247         enum resolve_mode resolve_mode;
 248 };
 249
 250 static const char * const btf_kind_str[NR_BTF_KINDS] = {
 251         [BTF_KIND_UNKN]         = "UNKNOWN",
 252         [BTF_KIND_INT]          = "INT",
 253         [BTF_KIND_PTR]          = "PTR",
 254         [BTF_KIND_ARRAY]        = "ARRAY",
 255         [BTF_KIND_STRUCT]       = "STRUCT",
 256         [BTF_KIND_UNION]        = "UNION",
 257         [BTF_KIND_ENUM]         = "ENUM",
 258         [BTF_KIND_FWD]          = "FWD",
 259         [BTF_KIND_TYPEDEF]      = "TYPEDEF",
 260         [BTF_KIND_VOLATILE]     = "VOLATILE",
 261         [BTF_KIND_CONST]        = "CONST",
 262         [BTF_KIND_RESTRICT]     = "RESTRICT",
 263         [BTF_KIND_FUNC]         = "FUNC",
 264         [BTF_KIND_FUNC_PROTO]   = "FUNC_PROTO",
 265 };
 266
 267 struct btf_kind_operations {
 268         s32 (*check_meta)(struct btf_verifier_env *env,
 269                           const struct btf_type *t,
 270                           u32 meta_left);
 271         int (*resolve)(struct btf_verifier_env *env,
 272                        const struct resolve_vertex *v);
 273         int (*check_member)(struct btf_verifier_env *env,
 274                             const struct btf_type *struct_type,
 275                             const struct btf_member *member,
 276                             const struct btf_type *member_type);
 277         int (*check_kflag_member)(struct btf_verifier_env *env,
 278                                   const struct btf_type *struct_type,
 279                                   const struct btf_member *member,
 280                                   const struct btf_type *member_type);
 281         void (*log_details)(struct btf_verifier_env *env,
 282                             const struct btf_type *t);
 283         void (*seq_show)(const struct btf *btf, const struct btf_type *t,
 284                          u32 type_id, void *data, u8 bits_offsets,
 285                          struct seq_file *m);
 286 };
 287
 288 static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS];
 289 static struct btf_type btf_void;
 290
 291 static int btf_resolve(struct btf_verifier_env *env,
 292                        const struct btf_type *t, u32 type_id);
 293
 294 static bool btf_type_is_modifier(const struct btf_type *t)
 295 {
 296         /* Some of them is not strictly a C modifier
 297          * but they are grouped into the same bucket
 298          * for BTF concern:
 299          *   A type (t) that refers to another
 300          *   type through t->type AND its size cannot
 301          *   be determined without following the t->type.
 302          *
 303          * ptr does not fall into this bucket
 304          * because its size is always sizeof(void *).
 305          */
 306         switch (BTF_INFO_KIND(t->info)) {
 307         case BTF_KIND_TYPEDEF:
 308         case BTF_KIND_VOLATILE:
 309         case BTF_KIND_CONST:
 310         case BTF_KIND_RESTRICT:
 311                 return true;
 312         }
 313
 314         return false;
 315 }
 316
 317 static bool btf_type_is_void(const struct btf_type *t)
 318 {
 319         return t == &btf_void;
 320 }
 321
 322 static bool btf_type_is_fwd(const struct btf_type *t)
 323 {
 324         return BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
 325 }
 326
 327 static bool btf_type_is_func(const struct btf_type *t)
 328 {
 329         return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC;
 330 }
 331
 332 static bool btf_type_is_func_proto(const struct btf_type *t)
 333 {
 334         return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;
 335 }
 336
 337 static bool btf_type_nosize(const struct btf_type *t)
 338 {
 339         return btf_type_is_void(t) || btf_type_is_fwd(t) ||
 340                btf_type_is_func(t) || btf_type_is_func_proto(t);
 341 }
 342
 343 static bool btf_type_nosize_or_null(const struct btf_type *t)
 344 {
 345         return !t || btf_type_nosize(t);
 346 }
 347
 348 /* union is only a special case of struct:
 349  * all its offsetof(member) == 0
 350  */
 351 static bool btf_type_is_struct(const struct btf_type *t)
 352 {
 353         u8 kind = BTF_INFO_KIND(t->info);
 354
 355         return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION;
 356 }
 357
 358 static bool btf_type_is_array(const struct btf_type *t)
 359 {
 360         return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
 361 }
 362
 363 static bool btf_type_is_ptr(const struct btf_type *t)
 364 {
 365         return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
 366 }
 367
 368 static bool btf_type_is_int(const struct btf_type *t)
 369 {
 370         return BTF_INFO_KIND(t->info) == BTF_KIND_INT;
 371 }
 372
 373 /* What types need to be resolved?
 374  *
 375  * btf_type_is_modifier() is an obvious one.
 376  *
 377  * btf_type_is_struct() because its member refers to
 378  * another type (through member->type).
 379
 380  * btf_type_is_array() because its element (array->type)
 381  * refers to another type.  Array can be thought of a
 382  * special case of struct while array just has the same
 383  * member-type repeated by array->nelems of times.
 384  */
 385 static bool btf_type_needs_resolve(const struct btf_type *t)
 386 {
 387         return btf_type_is_modifier(t) ||
 388                 btf_type_is_ptr(t) ||
 389                 btf_type_is_struct(t) ||
 390                 btf_type_is_array(t);
 391 }
 392
 393 /* t->size can be used */
 394 static bool btf_type_has_size(const struct btf_type *t)
 395 {
 396         switch (BTF_INFO_KIND(t->info)) {
 397         case BTF_KIND_INT:
 398         case BTF_KIND_STRUCT:
 399         case BTF_KIND_UNION:
 400         case BTF_KIND_ENUM:
 401                 return true;
 402         }
 403
 404         return false;
 405 }
 406
 407 static const char *btf_int_encoding_str(u8 encoding)
 408 {
 409         if (encoding == 0)
 410                 return "(none)";
 411         else if (encoding == BTF_INT_SIGNED)
 412                 return "SIGNED";
 413         else if (encoding == BTF_INT_CHAR)
 414                 return "CHAR";
 415         else if (encoding == BTF_INT_BOOL)
 416                 return "BOOL";
 417         else
 418                 return "UNKN";
 419 }
 420
 421 static u16 btf_type_vlen(const struct btf_type *t)
 422 {
 423         return BTF_INFO_VLEN(t->info);
 424 }
 425
 426 static bool btf_type_kflag(const struct btf_type *t)
 427 {
 428         return BTF_INFO_KFLAG(t->info);
 429 }
 430
 431 static u32 btf_member_bit_offset(const struct btf_type *struct_type,
 432                              const struct btf_member *member)
 433 {
 434         return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset)
 435                                            : member->offset;
 436 }
 437
 438 static u32 btf_member_bitfield_size(const struct btf_type *struct_type,
 439                                     const struct btf_member *member)
 440 {
 441         return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset)
 442                                            : 0;
 443 }
 444
 445 static u32 btf_type_int(const struct btf_type *t)
 446 {
 447         return *(u32 *)(t + 1);
 448 }
 449
 450 static const struct btf_array *btf_type_array(const struct btf_type *t)
 451 {
 452         return (const struct btf_array *)(t + 1);
 453 }
 454
 455 static const struct btf_member *btf_type_member(const struct btf_type *t)
 456 {
 457         return (const struct btf_member *)(t + 1);
 458 }
 459
 460 static const struct btf_enum *btf_type_enum(const struct btf_type *t)
 461 {
 462         return (const struct btf_enum *)(t + 1);
 463 }
 464
 465 static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
 466 {
 467         return kind_ops[BTF_INFO_KIND(t->info)];
 468 }
 469
 470 static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
 471 {
 472         return BTF_STR_OFFSET_VALID(offset) &&
 473                 offset < btf->hdr.str_len;
 474 }
 475
 476 /* Only C-style identifier is permitted. This can be relaxed if
 477  * necessary.
 478  */
 479 static bool btf_name_valid_identifier(const struct btf *btf, u32 offset)
 480 {
 481         /* offset must be valid */
 482         const char *src = &btf->strings[offset];
 483         const char *src_limit;
 484
 485         if (!isalpha(*src) && *src != '_')
 486                 return false;
 487
 488         /* set a limit on identifier length */
 489         src_limit = src + KSYM_NAME_LEN;
 490         src++;
 491         while (*src && src < src_limit) {
 492                 if (!isalnum(*src) && *src != '_')
 493                         return false;
 494                 src++;
 495         }
 496
 497         return !*src;
 498 }
 499
 500 static const char *__btf_name_by_offset(const struct btf *btf, u32 offset)
 501 {
 502         if (!offset)
 503                 return "(anon)";
 504         else if (offset < btf->hdr.str_len)
 505                 return &btf->strings[offset];
 506         else
 507                 return "(invalid-name-offset)";
 508 }
 509
 510 const char *btf_name_by_offset(const struct btf *btf, u32 offset)
 511 {
 512         if (offset < btf->hdr.str_len)
 513                 return &btf->strings[offset];
 514
 515         return NULL;
 516 }
 517
 518 const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
 519 {
 520         if (type_id > btf->nr_types)
 521                 return NULL;
 522
 523         return btf->types[type_id];
 524 }
 525
 526 /*
 527  * Regular int is not a bit field and it must be either
 528  * u8/u16/u32/u64.
 529  */
 530 static bool btf_type_int_is_regular(const struct btf_type *t)
 531 {
 532         u8 nr_bits, nr_bytes;
 533         u32 int_data;
 534
 535         int_data = btf_type_int(t);
 536         nr_bits = BTF_INT_BITS(int_data);
 537         nr_bytes = BITS_ROUNDUP_BYTES(nr_bits);
 538         if (BITS_PER_BYTE_MASKED(nr_bits) ||
 539             BTF_INT_OFFSET(int_data) ||
 540             (nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) &&
 541              nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64))) {
 542                 return false;
 543         }
 544
 545         return true;
 546 }
 547
 548 /*
 549  * Check that given struct member is a regular int with expected
 550  * offset and size.
 551  */
 552 bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,
 553                            const struct btf_member *m,
 554                            u32 expected_offset, u32 expected_size)
 555 {
 556         const struct btf_type *t;
 557         u32 id, int_data;
 558         u8 nr_bits;
 559
 560         id = m->type;
 561         t = btf_type_id_size(btf, &id, NULL);
 562         if (!t || !btf_type_is_int(t))
 563                 return false;
 564
 565         int_data = btf_type_int(t);
 566         nr_bits = BTF_INT_BITS(int_data);
 567         if (btf_type_kflag(s)) {
 568                 u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset);
 569                 u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset);
 570
 571                 /* if kflag set, int should be a regular int and
 572                  * bit offset should be at byte boundary.
 573                  */
 574                 return !bitfield_size &&
 575                        BITS_ROUNDUP_BYTES(bit_offset) == expected_offset &&
 576                        BITS_ROUNDUP_BYTES(nr_bits) == expected_size;
 577         }
 578
 579         if (BTF_INT_OFFSET(int_data) ||
 580             BITS_PER_BYTE_MASKED(m->offset) ||
 581             BITS_ROUNDUP_BYTES(m->offset) != expected_offset ||
 582             BITS_PER_BYTE_MASKED(nr_bits) ||
 583             BITS_ROUNDUP_BYTES(nr_bits) != expected_size)
 584                 return false;
 585
 586         return true;
 587 }
 588
 589 __printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log,
 590                                               const char *fmt, ...)
 591 {
 592         va_list args;
 593
 594         va_start(args, fmt);
 595         bpf_verifier_vlog(log, fmt, args);
 596         va_end(args);
 597 }
 598
 599 __printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env,
 600                                             const char *fmt, ...)
 601 {
 602         struct bpf_verifier_log *log = &env->log;
 603         va_list args;
 604
 605         if (!bpf_verifier_log_needed(log))
 606                 return;
 607
 608         va_start(args, fmt);
 609         bpf_verifier_vlog(log, fmt, args);
 610         va_end(args);
 611 }
 612
 613 __printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,
 614                                                    const struct btf_type *t,
 615                                                    bool log_details,
 616                                                    const char *fmt, ...)
 617 {
 618         struct bpf_verifier_log *log = &env->log;
 619         u8 kind = BTF_INFO_KIND(t->info);
 620         struct btf *btf = env->btf;
 621         va_list args;
 622
 623         if (!bpf_verifier_log_needed(log))
 624                 return;
 625
 626         __btf_verifier_log(log, "[%u] %s %s%s",
 627                            env->log_type_id,
 628                            btf_kind_str[kind],
 629                            __btf_name_by_offset(btf, t->name_off),
 630                            log_details ? " " : "");
 631
 632         if (log_details)
 633                 btf_type_ops(t)->log_details(env, t);
 634
 635         if (fmt && *fmt) {
 636                 __btf_verifier_log(log, " ");
 637                 va_start(args, fmt);
 638                 bpf_verifier_vlog(log, fmt, args);
 639                 va_end(args);
 640         }
 641
 642         __btf_verifier_log(log, "\n");
 643 }
 644
 645 #define btf_verifier_log_type(env, t, ...) \
 646         __btf_verifier_log_type((env), (t), true, __VA_ARGS__)
 647 #define btf_verifier_log_basic(env, t, ...) \
 648         __btf_verifier_log_type((env), (t), false, __VA_ARGS__)
 649
 650 __printf(4, 5)
 651 static void btf_verifier_log_member(struct btf_verifier_env *env,
 652                                     const struct btf_type *struct_type,
 653                                     const struct btf_member *member,
 654                                     const char *fmt, ...)
 655 {
 656         struct bpf_verifier_log *log = &env->log;
 657         struct btf *btf = env->btf;
 658         va_list args;
 659
 660         if (!bpf_verifier_log_needed(log))
 661                 return;
 662
 663         /* The CHECK_META phase already did a btf dump.
 664          *
 665          * If member is logged again, it must hit an error in
 666          * parsing this member.  It is useful to print out which
 667          * struct this member belongs to.
 668          */
 669         if (env->phase != CHECK_META)
 670                 btf_verifier_log_type(env, struct_type, NULL);
 671
 672         if (btf_type_kflag(struct_type))
 673                 __btf_verifier_log(log,
 674                                    "\t%s type_id=%u bitfield_size=%u bits_offset=%u",
 675                                    __btf_name_by_offset(btf, member->name_off),
 676                                    member->type,
 677                                    BTF_MEMBER_BITFIELD_SIZE(member->offset),
 678                                    BTF_MEMBER_BIT_OFFSET(member->offset));
 679         else
 680                 __btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u",
 681                                    __btf_name_by_offset(btf, member->name_off),
 682                                    member->type, member->offset);
 683
 684         if (fmt && *fmt) {
 685                 __btf_verifier_log(log, " ");
 686                 va_start(args, fmt);
 687                 bpf_verifier_vlog(log, fmt, args);
 688                 va_end(args);
 689         }
 690
 691         __btf_verifier_log(log, "\n");
 692 }
 693
 694 static void btf_verifier_log_hdr(struct btf_verifier_env *env,
 695                                  u32 btf_data_size)
 696 {
 697         struct bpf_verifier_log *log = &env->log;
 698         const struct btf *btf = env->btf;
 699         const struct btf_header *hdr;
 700
 701         if (!bpf_verifier_log_needed(log))
 702                 return;
 703
 704         hdr = &btf->hdr;
 705         __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic);
 706         __btf_verifier_log(log, "version: %u\n", hdr->version);
 707         __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags);
 708         __btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len);
 709         __btf_verifier_log(log, "type_off: %u\n", hdr->type_off);
 710         __btf_verifier_log(log, "type_len: %u\n", hdr->type_len);
 711         __btf_verifier_log(log, "str_off: %u\n", hdr->str_off);
 712         __btf_verifier_log(log, "str_len: %u\n", hdr->str_len);
 713         __btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size);
 714 }
 715
 716 static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)
 717 {
 718         struct btf *btf = env->btf;
 719
 720         /* < 2 because +1 for btf_void which is always in btf->types[0].
 721          * btf_void is not accounted in btf->nr_types because btf_void
 722          * does not come from the BTF file.
 723          */
 724         if (btf->types_size - btf->nr_types < 2) {
 725                 /* Expand 'types' array */
 726
 727                 struct btf_type **new_types;
 728                 u32 expand_by, new_size;
 729
 730                 if (btf->types_size == BTF_MAX_TYPE) {
 731                         btf_verifier_log(env, "Exceeded max num of types");
 732                         return -E2BIG;
 733                 }
 734
 735                 expand_by = max_t(u32, btf->types_size >> 2, 16);
 736                 new_size = min_t(u32, BTF_MAX_TYPE,
 737                                  btf->types_size + expand_by);
 738
 739                 new_types = kvcalloc(new_size, sizeof(*new_types),
 740                                      GFP_KERNEL | __GFP_NOWARN);
 741                 if (!new_types)
 742                         return -ENOMEM;
 743
 744                 if (btf->nr_types == 0)
 745                         new_types[0] = &btf_void;
 746                 else
 747                         memcpy(new_types, btf->types,
 748                                sizeof(*btf->types) * (btf->nr_types + 1));
 749
 750                 kvfree(btf->types);
 751                 btf->types = new_types;
 752                 btf->types_size = new_size;
 753         }
 754
 755         btf->types[++(btf->nr_types)] = t;
 756
 757         return 0;
 758 }
 759
 760 static int btf_alloc_id(struct btf *btf)
 761 {
 762         int id;
 763
 764         idr_preload(GFP_KERNEL);
 765         spin_lock_bh(&btf_idr_lock);
 766         id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC);
 767         if (id > 0)
 768                 btf->id = id;
 769         spin_unlock_bh(&btf_idr_lock);
 770         idr_preload_end();
 771
 772         if (WARN_ON_ONCE(!id))
 773                 return -ENOSPC;
 774
 775         return id > 0 ? 0 : id;
 776 }
 777
 778 static void btf_free_id(struct btf *btf)
 779 {
 780         unsigned long flags;
 781
 782         /*
 783          * In map-in-map, calling map_delete_elem() on outer
 784          * map will call bpf_map_put on the inner map.
 785          * It will then eventually call btf_free_id()
 786          * on the inner map.  Some of the map_delete_elem()
 787          * implementation may have irq disabled, so
 788          * we need to use the _irqsave() version instead
 789          * of the _bh() version.
 790          */
 791         spin_lock_irqsave(&btf_idr_lock, flags);
 792         idr_remove(&btf_idr, btf->id);
 793         spin_unlock_irqrestore(&btf_idr_lock, flags);
 794 }
 795
 796 static void btf_free(struct btf *btf)
 797 {
 798         kvfree(btf->types);
 799         kvfree(btf->resolved_sizes);
 800         kvfree(btf->resolved_ids);
 801         kvfree(btf->data);
 802         kfree(btf);
 803 }
 804
 805 static void btf_free_rcu(struct rcu_head *rcu)
 806 {
 807         struct btf *btf = container_of(rcu, struct btf, rcu);
 808
 809         btf_free(btf);
 810 }
 811
 812 void btf_put(struct btf *btf)
 813 {
 814         if (btf && refcount_dec_and_test(&btf->refcnt)) {
 815                 btf_free_id(btf);
 816                 call_rcu(&btf->rcu, btf_free_rcu);
 817         }
 818 }
 819
 820 static int env_resolve_init(struct btf_verifier_env *env)
 821 {
 822         struct btf *btf = env->btf;
 823         u32 nr_types = btf->nr_types;
 824         u32 *resolved_sizes = NULL;
 825         u32 *resolved_ids = NULL;
 826         u8 *visit_states = NULL;
 827
 828         /* +1 for btf_void */
 829         resolved_sizes = kvcalloc(nr_types + 1, sizeof(*resolved_sizes),
 830                                   GFP_KERNEL | __GFP_NOWARN);
 831         if (!resolved_sizes)
 832                 goto nomem;
 833
 834         resolved_ids = kvcalloc(nr_types + 1, sizeof(*resolved_ids),
 835                                 GFP_KERNEL | __GFP_NOWARN);
 836         if (!resolved_ids)
 837                 goto nomem;
 838
 839         visit_states = kvcalloc(nr_types + 1, sizeof(*visit_states),
 840                                 GFP_KERNEL | __GFP_NOWARN);
 841         if (!visit_states)
 842                 goto nomem;
 843
 844         btf->resolved_sizes = resolved_sizes;
 845         btf->resolved_ids = resolved_ids;
 846         env->visit_states = visit_states;
 847
 848         return 0;
 849
 850 nomem:
 851         kvfree(resolved_sizes);
 852         kvfree(resolved_ids);
 853         kvfree(visit_states);
 854         return -ENOMEM;
 855 }
 856
 857 static void btf_verifier_env_free(struct btf_verifier_env *env)
 858 {
 859         kvfree(env->visit_states);
 860         kfree(env);
 861 }
 862
 863 static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,
 864                                      const struct btf_type *next_type)
 865 {
 866         switch (env->resolve_mode) {
 867         case RESOLVE_TBD:
 868                 /* int, enum or void is a sink */
 869                 return !btf_type_needs_resolve(next_type);
 870         case RESOLVE_PTR:
 871                 /* int, enum, void, struct, array, func or func_proto is a sink
 872                  * for ptr
 873                  */
 874                 return !btf_type_is_modifier(next_type) &&
 875                         !btf_type_is_ptr(next_type);
 876         case RESOLVE_STRUCT_OR_ARRAY:
 877                 /* int, enum, void, ptr, func or func_proto is a sink
 878                  * for struct and array
 879                  */
 880                 return !btf_type_is_modifier(next_type) &&
 881                         !btf_type_is_array(next_type) &&
 882                         !btf_type_is_struct(next_type);
 883         default:
 884                 BUG();
 885         }
 886 }
 887
 888 static bool env_type_is_resolved(const struct btf_verifier_env *env,
 889                                  u32 type_id)
 890 {
 891         return env->visit_states[type_id] == RESOLVED;
 892 }
 893
 894 static int env_stack_push(struct btf_verifier_env *env,
 895                           const struct btf_type *t, u32 type_id)
 896 {
 897         struct resolve_vertex *v;
 898
 899         if (env->top_stack == MAX_RESOLVE_DEPTH)
 900                 return -E2BIG;
 901
 902         if (env->visit_states[type_id] != NOT_VISITED)
 903                 return -EEXIST;
 904
 905         env->visit_states[type_id] = VISITED;
 906
 907         v = &env->stack[env->top_stack++];
 908         v->t = t;
 909         v->type_id = type_id;
 910         v->next_member = 0;
 911
 912         if (env->resolve_mode == RESOLVE_TBD) {
 913                 if (btf_type_is_ptr(t))
 914                         env->resolve_mode = RESOLVE_PTR;
 915                 else if (btf_type_is_struct(t) || btf_type_is_array(t))
 916                         env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY;
 917         }
 918
 919         return 0;
 920 }
 921
 922 static void env_stack_set_next_member(struct btf_verifier_env *env,
 923                                       u16 next_member)
 924 {
 925         env->stack[env->top_stack - 1].next_member = next_member;
 926 }
 927
 928 static void env_stack_pop_resolved(struct btf_verifier_env *env,
 929                                    u32 resolved_type_id,
 930                                    u32 resolved_size)
 931 {
 932         u32 type_id = env->stack[--(env->top_stack)].type_id;
 933         struct btf *btf = env->btf;
 934
 935         btf->resolved_sizes[type_id] = resolved_size;
 936         btf->resolved_ids[type_id] = resolved_type_id;
 937         env->visit_states[type_id] = RESOLVED;
 938 }
 939
 940 static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env)
 941 {
 942         return env->top_stack ? &env->stack[env->top_stack - 1] : NULL;
 943 }
 944
 945 /* The input param "type_id" must point to a needs_resolve type */
 946 static const struct btf_type *btf_type_id_resolve(const struct btf *btf,
 947                                                   u32 *type_id)
 948 {
 949         *type_id = btf->resolved_ids[*type_id];
 950         return btf_type_by_id(btf, *type_id);
 951 }
 952
 953 const struct btf_type *btf_type_id_size(const struct btf *btf,
 954                                         u32 *type_id, u32 *ret_size)
 955 {
 956         const struct btf_type *size_type;
 957         u32 size_type_id = *type_id;
 958         u32 size = 0;
 959
 960         size_type = btf_type_by_id(btf, size_type_id);
 961         if (btf_type_nosize_or_null(size_type))
 962                 return NULL;
 963
 964         if (btf_type_has_size(size_type)) {
 965                 size = size_type->size;
 966         } else if (btf_type_is_array(size_type)) {
 967                 size = btf->resolved_sizes[size_type_id];
 968         } else if (btf_type_is_ptr(size_type)) {
 969                 size = sizeof(void *);
 970         } else {
 971                 if (WARN_ON_ONCE(!btf_type_is_modifier(size_type)))
 972                         return NULL;
 973
 974                 size = btf->resolved_sizes[size_type_id];
 975                 size_type_id = btf->resolved_ids[size_type_id];
 976                 size_type = btf_type_by_id(btf, size_type_id);
 977                 if (btf_type_nosize_or_null(size_type))
 978                         return NULL;
 979         }
 980
 981         *type_id = size_type_id;
 982         if (ret_size)
 983                 *ret_size = size;
 984
 985         return size_type;
 986 }
 987
 988 static int btf_df_check_member(struct btf_verifier_env *env,
 989                                const struct btf_type *struct_type,
 990                                const struct btf_member *member,
 991                                const struct btf_type *member_type)
 992 {
 993         btf_verifier_log_basic(env, struct_type,
 994                                "Unsupported check_member");
 995         return -EINVAL;
 996 }
 997
 998 static int btf_df_check_kflag_member(struct btf_verifier_env *env,
 999                                      const struct btf_type *struct_type,
1000                                      const struct btf_member *member,
1001                                      const struct btf_type *member_type)
1002 {
1003         btf_verifier_log_basic(env, struct_type,
1004                                "Unsupported check_kflag_member");
1005         return -EINVAL;
1006 }
1007
1008 /* Used for ptr, array and struct/union type members.
1009  * int, enum and modifier types have their specific callback functions.
1010  */
1011 static int btf_generic_check_kflag_member(struct btf_verifier_env *env,
1012                                           const struct btf_type *struct_type,
1013                                           const struct btf_member *member,
1014                                           const struct btf_type *member_type)
1015 {
1016         if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) {
1017                 btf_verifier_log_member(env, struct_type, member,
1018                                         "Invalid member bitfield_size");
1019                 return -EINVAL;
1020         }
1021
1022         /* bitfield size is 0, so member->offset represents bit offset only.
1023          * It is safe to call non kflag check_member variants.
1024          */
1025         return btf_type_ops(member_type)->check_member(env, struct_type,
1026                                                        member,
1027                                                        member_type);
1028 }
1029
1030 static int btf_df_resolve(struct btf_verifier_env *env,
1031                           const struct resolve_vertex *v)
1032 {
1033         btf_verifier_log_basic(env, v->t, "Unsupported resolve");
1034         return -EINVAL;
1035 }
1036
1037 static void btf_df_seq_show(const struct btf *btf, const struct btf_type *t,
1038                             u32 type_id, void *data, u8 bits_offsets,
1039                             struct seq_file *m)
1040 {
1041         seq_printf(m, "<unsupported kind:%u>", BTF_INFO_KIND(t->info));
1042 }
1043
1044 static int btf_int_check_member(struct btf_verifier_env *env,
1045                                 const struct btf_type *struct_type,
1046                                 const struct btf_member *member,
1047                                 const struct btf_type *member_type)
1048 {
1049         u32 int_data = btf_type_int(member_type);
1050         u32 struct_bits_off = member->offset;
1051         u32 struct_size = struct_type->size;
1052         u32 nr_copy_bits;
1053         u32 bytes_offset;
1054
1055         if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) {
1056                 btf_verifier_log_member(env, struct_type, member,
1057                                         "bits_offset exceeds U32_MAX");
1058                 return -EINVAL;
1059         }
1060
1061         struct_bits_off += BTF_INT_OFFSET(int_data);
1062         bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1063         nr_copy_bits = BTF_INT_BITS(int_data) +
1064                 BITS_PER_BYTE_MASKED(struct_bits_off);
1065
1066         if (nr_copy_bits > BITS_PER_U64) {
1067                 btf_verifier_log_member(env, struct_type, member,
1068                                         "nr_copy_bits exceeds 64");
1069                 return -EINVAL;
1070         }
1071
1072         if (struct_size < bytes_offset ||
1073             struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
1074                 btf_verifier_log_member(env, struct_type, member,
1075                                         "Member exceeds struct_size");
1076                 return -EINVAL;
1077         }
1078
1079         return 0;
1080 }
1081
1082 static int btf_int_check_kflag_member(struct btf_verifier_env *env,
1083                                       const struct btf_type *struct_type,
1084                                       const struct btf_member *member,
1085                                       const struct btf_type *member_type)
1086 {
1087         u32 struct_bits_off, nr_bits, nr_int_data_bits, bytes_offset;
1088         u32 int_data = btf_type_int(member_type);
1089         u32 struct_size = struct_type->size;
1090         u32 nr_copy_bits;
1091
1092         /* a regular int type is required for the kflag int member */
1093         if (!btf_type_int_is_regular(member_type)) {
1094                 btf_verifier_log_member(env, struct_type, member,
1095                                         "Invalid member base type");
1096                 return -EINVAL;
1097         }
1098
1099         /* check sanity of bitfield size */
1100         nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset);
1101         struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset);
1102         nr_int_data_bits = BTF_INT_BITS(int_data);
1103         if (!nr_bits) {
1104                 /* Not a bitfield member, member offset must be at byte
1105                  * boundary.
1106                  */
1107                 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1108                         btf_verifier_log_member(env, struct_type, member,
1109                                                 "Invalid member offset");
1110                         return -EINVAL;
1111                 }
1112
1113                 nr_bits = nr_int_data_bits;
1114         } else if (nr_bits > nr_int_data_bits) {
1115                 btf_verifier_log_member(env, struct_type, member,
1116                                         "Invalid member bitfield_size");
1117                 return -EINVAL;
1118         }
1119
1120         bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1121         nr_copy_bits = nr_bits + BITS_PER_BYTE_MASKED(struct_bits_off);
1122         if (nr_copy_bits > BITS_PER_U64) {
1123                 btf_verifier_log_member(env, struct_type, member,
1124                                         "nr_copy_bits exceeds 64");
1125                 return -EINVAL;
1126         }
1127
1128         if (struct_size < bytes_offset ||
1129             struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
1130                 btf_verifier_log_member(env, struct_type, member,
1131                                         "Member exceeds struct_size");
1132                 return -EINVAL;
1133         }
1134
1135         return 0;
1136 }
1137
1138 static s32 btf_int_check_meta(struct btf_verifier_env *env,
1139                               const struct btf_type *t,
1140                               u32 meta_left)
1141 {
1142         u32 int_data, nr_bits, meta_needed = sizeof(int_data);
1143         u16 encoding;
1144
1145         if (meta_left < meta_needed) {
1146                 btf_verifier_log_basic(env, t,
1147                                        "meta_left:%u meta_needed:%u",
1148                                        meta_left, meta_needed);
1149                 return -EINVAL;
1150         }
1151
1152         if (btf_type_vlen(t)) {
1153                 btf_verifier_log_type(env, t, "vlen != 0");
1154                 return -EINVAL;
1155         }
1156
1157         if (btf_type_kflag(t)) {
1158                 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
1159                 return -EINVAL;
1160         }
1161
1162         int_data = btf_type_int(t);
1163         if (int_data & ~BTF_INT_MASK) {
1164                 btf_verifier_log_basic(env, t, "Invalid int_data:%x",
1165                                        int_data);
1166                 return -EINVAL;
1167         }
1168
1169         nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data);
1170
1171         if (nr_bits > BITS_PER_U64) {
1172                 btf_verifier_log_type(env, t, "nr_bits exceeds %zu",
1173                                       BITS_PER_U64);
1174                 return -EINVAL;
1175         }
1176
1177         if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) {
1178                 btf_verifier_log_type(env, t, "nr_bits exceeds type_size");
1179                 return -EINVAL;
1180         }
1181
1182         /*
1183          * Only one of the encoding bits is allowed and it
1184          * should be sufficient for the pretty print purpose (i.e. decoding).
1185          * Multiple bits can be allowed later if it is found
1186          * to be insufficient.
1187          */
1188         encoding = BTF_INT_ENCODING(int_data);
1189         if (encoding &&
1190             encoding != BTF_INT_SIGNED &&
1191             encoding != BTF_INT_CHAR &&
1192             encoding != BTF_INT_BOOL) {
1193                 btf_verifier_log_type(env, t, "Unsupported encoding");
1194                 return -ENOTSUPP;
1195         }
1196
1197         btf_verifier_log_type(env, t, NULL);
1198
1199         return meta_needed;
1200 }
1201
1202 static void btf_int_log(struct btf_verifier_env *env,
1203                         const struct btf_type *t)
1204 {
1205         int int_data = btf_type_int(t);
1206
1207         btf_verifier_log(env,
1208                          "size=%u bits_offset=%u nr_bits=%u encoding=%s",
1209                          t->size, BTF_INT_OFFSET(int_data),
1210                          BTF_INT_BITS(int_data),
1211                          btf_int_encoding_str(BTF_INT_ENCODING(int_data)));
1212 }
1213
1214 static void btf_bitfield_seq_show(void *data, u8 bits_offset,
1215                                   u8 nr_bits, struct seq_file *m)
1216 {
1217         u16 left_shift_bits, right_shift_bits;
1218         u8 nr_copy_bytes;
1219         u8 nr_copy_bits;
1220         u64 print_num;
1221
1222         nr_copy_bits = nr_bits + bits_offset;
1223         nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits);
1224
1225         print_num = 0;
1226         memcpy(&print_num, data, nr_copy_bytes);
1227
1228 #ifdef __BIG_ENDIAN_BITFIELD
1229         left_shift_bits = bits_offset;
1230 #else
1231         left_shift_bits = BITS_PER_U64 - nr_copy_bits;
1232 #endif
1233         right_shift_bits = BITS_PER_U64 - nr_bits;
1234
1235         print_num <<= left_shift_bits;
1236         print_num >>= right_shift_bits;
1237
1238         seq_printf(m, "0x%llx", print_num);
1239 }
1240
1241
1242 static void btf_int_bits_seq_show(const struct btf *btf,
1243                                   const struct btf_type *t,
1244                                   void *data, u8 bits_offset,
1245                                   struct seq_file *m)
1246 {
1247         u32 int_data = btf_type_int(t);
1248         u8 nr_bits = BTF_INT_BITS(int_data);
1249         u8 total_bits_offset;
1250
1251         /*
1252          * bits_offset is at most 7.
1253          * BTF_INT_OFFSET() cannot exceed 64 bits.
1254          */
1255         total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);
1256         data += BITS_ROUNDDOWN_BYTES(total_bits_offset);
1257         bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);
1258         btf_bitfield_seq_show(data, bits_offset, nr_bits, m);
1259 }
1260
1261 static void btf_int_seq_show(const struct btf *btf, const struct btf_type *t,
1262                              u32 type_id, void *data, u8 bits_offset,
1263                              struct seq_file *m)
1264 {
1265         u32 int_data = btf_type_int(t);
1266         u8 encoding = BTF_INT_ENCODING(int_data);
1267         bool sign = encoding & BTF_INT_SIGNED;
1268         u8 nr_bits = BTF_INT_BITS(int_data);
1269
1270         if (bits_offset || BTF_INT_OFFSET(int_data) ||
1271             BITS_PER_BYTE_MASKED(nr_bits)) {
1272                 btf_int_bits_seq_show(btf, t, data, bits_offset, m);
1273                 return;
1274         }
1275
1276         switch (nr_bits) {
1277         case 64:
1278                 if (sign)
1279                         seq_printf(m, "%lld", *(s64 *)data);
1280                 else
1281                         seq_printf(m, "%llu", *(u64 *)data);
1282                 break;
1283         case 32:
1284                 if (sign)
1285                         seq_printf(m, "%d", *(s32 *)data);
1286                 else
1287                         seq_printf(m, "%u", *(u32 *)data);
1288                 break;
1289         case 16:
1290                 if (sign)
1291                         seq_printf(m, "%d", *(s16 *)data);
1292                 else
1293                         seq_printf(m, "%u", *(u16 *)data);
1294                 break;
1295         case 8:
1296                 if (sign)
1297                         seq_printf(m, "%d", *(s8 *)data);
1298                 else
1299                         seq_printf(m, "%u", *(u8 *)data);
1300                 break;
1301         default:
1302                 btf_int_bits_seq_show(btf, t, data, bits_offset, m);
1303         }
1304 }
1305
1306 static const struct btf_kind_operations int_ops = {
1307         .check_meta = btf_int_check_meta,
1308         .resolve = btf_df_resolve,
1309         .check_member = btf_int_check_member,
1310         .check_kflag_member = btf_int_check_kflag_member,
1311         .log_details = btf_int_log,
1312         .seq_show = btf_int_seq_show,
1313 };
1314
1315 static int btf_modifier_check_member(struct btf_verifier_env *env,
1316                                      const struct btf_type *struct_type,
1317                                      const struct btf_member *member,
1318                                      const struct btf_type *member_type)
1319 {
1320         const struct btf_type *resolved_type;
1321         u32 resolved_type_id = member->type;
1322         struct btf_member resolved_member;
1323         struct btf *btf = env->btf;
1324
1325         resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
1326         if (!resolved_type) {
1327                 btf_verifier_log_member(env, struct_type, member,
1328                                         "Invalid member");
1329                 return -EINVAL;
1330         }
1331
1332         resolved_member = *member;
1333         resolved_member.type = resolved_type_id;
1334
1335         return btf_type_ops(resolved_type)->check_member(env, struct_type,
1336                                                          &resolved_member,
1337                                                          resolved_type);
1338 }
1339
1340 static int btf_modifier_check_kflag_member(struct btf_verifier_env *env,
1341                                            const struct btf_type *struct_type,
1342                                            const struct btf_member *member,
1343                                            const struct btf_type *member_type)
1344 {
1345         const struct btf_type *resolved_type;
1346         u32 resolved_type_id = member->type;
1347         struct btf_member resolved_member;
1348         struct btf *btf = env->btf;
1349
1350         resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
1351         if (!resolved_type) {
1352                 btf_verifier_log_member(env, struct_type, member,
1353                                         "Invalid member");
1354                 return -EINVAL;
1355         }
1356
1357         resolved_member = *member;
1358         resolved_member.type = resolved_type_id;
1359
1360         return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type,
1361                                                                &resolved_member,
1362                                                                resolved_type);
1363 }
1364
1365 static int btf_ptr_check_member(struct btf_verifier_env *env,
1366                                 const struct btf_type *struct_type,
1367                                 const struct btf_member *member,
1368                                 const struct btf_type *member_type)
1369 {
1370         u32 struct_size, struct_bits_off, bytes_offset;
1371
1372         struct_size = struct_type->size;
1373         struct_bits_off = member->offset;
1374         bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1375
1376         if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1377                 btf_verifier_log_member(env, struct_type, member,
1378                                         "Member is not byte aligned");
1379                 return -EINVAL;
1380         }
1381
1382         if (struct_size - bytes_offset < sizeof(void *)) {
1383                 btf_verifier_log_member(env, struct_type, member,
1384                                         "Member exceeds struct_size");
1385                 return -EINVAL;
1386         }
1387
1388         return 0;
1389 }
1390
1391 static int btf_ref_type_check_meta(struct btf_verifier_env *env,
1392                                    const struct btf_type *t,
1393                                    u32 meta_left)
1394 {
1395         if (btf_type_vlen(t)) {
1396                 btf_verifier_log_type(env, t, "vlen != 0");
1397                 return -EINVAL;
1398         }
1399
1400         if (btf_type_kflag(t)) {
1401                 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
1402                 return -EINVAL;
1403         }
1404
1405         if (!BTF_TYPE_ID_VALID(t->type)) {
1406                 btf_verifier_log_type(env, t, "Invalid type_id");
1407                 return -EINVAL;
1408         }
1409
1410         /* typedef type must have a valid name, and other ref types,
1411          * volatile, const, restrict, should have a null name.
1412          */
1413         if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) {
1414                 if (!t->name_off ||
1415                     !btf_name_valid_identifier(env->btf, t->name_off)) {
1416                         btf_verifier_log_type(env, t, "Invalid name");
1417                         return -EINVAL;
1418                 }
1419         } else {
1420                 if (t->name_off) {
1421                         btf_verifier_log_type(env, t, "Invalid name");
1422                         return -EINVAL;
1423                 }
1424         }
1425
1426         btf_verifier_log_type(env, t, NULL);
1427
1428         return 0;
1429 }
1430
1431 static int btf_modifier_resolve(struct btf_verifier_env *env,
1432                                 const struct resolve_vertex *v)
1433 {
1434         const struct btf_type *t = v->t;
1435         const struct btf_type *next_type;
1436         u32 next_type_id = t->type;
1437         struct btf *btf = env->btf;
1438         u32 next_type_size = 0;
1439
1440         next_type = btf_type_by_id(btf, next_type_id);
1441         if (!next_type) {
1442                 btf_verifier_log_type(env, v->t, "Invalid type_id");
1443                 return -EINVAL;
1444         }
1445
1446         if (!env_type_is_resolve_sink(env, next_type) &&
1447             !env_type_is_resolved(env, next_type_id))
1448                 return env_stack_push(env, next_type, next_type_id);
1449
1450         /* Figure out the resolved next_type_id with size.
1451          * They will be stored in the current modifier's
1452          * resolved_ids and resolved_sizes such that it can
1453          * save us a few type-following when we use it later (e.g. in
1454          * pretty print).
1455          */
1456         if (!btf_type_id_size(btf, &next_type_id, &next_type_size)) {
1457                 if (env_type_is_resolved(env, next_type_id))
1458                         next_type = btf_type_id_resolve(btf, &next_type_id);
1459
1460                 /* "typedef void new_void", "const void"...etc */
1461                 if (!btf_type_is_void(next_type) &&
1462                     !btf_type_is_fwd(next_type)) {
1463                         btf_verifier_log_type(env, v->t, "Invalid type_id");
1464                         return -EINVAL;
1465                 }
1466         }
1467
1468         env_stack_pop_resolved(env, next_type_id, next_type_size);
1469
1470         return 0;
1471 }
1472
1473 static int btf_ptr_resolve(struct btf_verifier_env *env,
1474                            const struct resolve_vertex *v)
1475 {
1476         const struct btf_type *next_type;
1477         const struct btf_type *t = v->t;
1478         u32 next_type_id = t->type;
1479         struct btf *btf = env->btf;
1480
1481         next_type = btf_type_by_id(btf, next_type_id);
1482         if (!next_type) {
1483                 btf_verifier_log_type(env, v->t, "Invalid type_id");
1484                 return -EINVAL;
1485         }
1486
1487         if (!env_type_is_resolve_sink(env, next_type) &&
1488             !env_type_is_resolved(env, next_type_id))
1489                 return env_stack_push(env, next_type, next_type_id);
1490
1491         /* If the modifier was RESOLVED during RESOLVE_STRUCT_OR_ARRAY,
1492          * the modifier may have stopped resolving when it was resolved
1493          * to a ptr (last-resolved-ptr).
1494          *
1495          * We now need to continue from the last-resolved-ptr to
1496          * ensure the last-resolved-ptr will not referring back to
1497          * the currenct ptr (t).
1498          */
1499         if (btf_type_is_modifier(next_type)) {
1500                 const struct btf_type *resolved_type;
1501                 u32 resolved_type_id;
1502
1503                 resolved_type_id = next_type_id;
1504                 resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
1505
1506                 if (btf_type_is_ptr(resolved_type) &&
1507                     !env_type_is_resolve_sink(env, resolved_type) &&
1508                     !env_type_is_resolved(env, resolved_type_id))
1509                         return env_stack_push(env, resolved_type,
1510                                               resolved_type_id);
1511         }
1512
1513         if (!btf_type_id_size(btf, &next_type_id, NULL)) {
1514                 if (env_type_is_resolved(env, next_type_id))
1515                         next_type = btf_type_id_resolve(btf, &next_type_id);
1516
1517                 if (!btf_type_is_void(next_type) &&
1518                     !btf_type_is_fwd(next_type) &&
1519                     !btf_type_is_func_proto(next_type)) {
1520                         btf_verifier_log_type(env, v->t, "Invalid type_id");
1521                         return -EINVAL;
1522                 }
1523         }
1524
1525         env_stack_pop_resolved(env, next_type_id, 0);
1526
1527         return 0;
1528 }
1529
1530 static void btf_modifier_seq_show(const struct btf *btf,
1531                                   const struct btf_type *t,
1532                                   u32 type_id, void *data,
1533                                   u8 bits_offset, struct seq_file *m)
1534 {
1535         t = btf_type_id_resolve(btf, &type_id);
1536
1537         btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m);
1538 }
1539
1540 static void btf_ptr_seq_show(const struct btf *btf, const struct btf_type *t,
1541                              u32 type_id, void *data, u8 bits_offset,
1542                              struct seq_file *m)
1543 {
1544         /* It is a hashed value */
1545         seq_printf(m, "%p", *(void **)data);
1546 }
1547
1548 static void btf_ref_type_log(struct btf_verifier_env *env,
1549                              const struct btf_type *t)
1550 {
1551         btf_verifier_log(env, "type_id=%u", t->type);
1552 }
1553
1554 static struct btf_kind_operations modifier_ops = {
1555         .check_meta = btf_ref_type_check_meta,
1556         .resolve = btf_modifier_resolve,
1557         .check_member = btf_modifier_check_member,
1558         .check_kflag_member = btf_modifier_check_kflag_member,
1559         .log_details = btf_ref_type_log,
1560         .seq_show = btf_modifier_seq_show,
1561 };
1562
1563 static struct btf_kind_operations ptr_ops = {
1564         .check_meta = btf_ref_type_check_meta,
1565         .resolve = btf_ptr_resolve,
1566         .check_member = btf_ptr_check_member,
1567         .check_kflag_member = btf_generic_check_kflag_member,
1568         .log_details = btf_ref_type_log,
1569         .seq_show = btf_ptr_seq_show,
1570 };
1571
1572 static s32 btf_fwd_check_meta(struct btf_verifier_env *env,
1573                               const struct btf_type *t,
1574                               u32 meta_left)
1575 {
1576         if (btf_type_vlen(t)) {
1577                 btf_verifier_log_type(env, t, "vlen != 0");
1578                 return -EINVAL;
1579         }
1580
1581         if (t->type) {
1582                 btf_verifier_log_type(env, t, "type != 0");
1583                 return -EINVAL;
1584         }
1585
1586         /* fwd type must have a valid name */
1587         if (!t->name_off ||
1588             !btf_name_valid_identifier(env->btf, t->name_off)) {
1589                 btf_verifier_log_type(env, t, "Invalid name");
1590                 return -EINVAL;
1591         }
1592
1593         btf_verifier_log_type(env, t, NULL);
1594
1595         return 0;
1596 }
1597
1598 static void btf_fwd_type_log(struct btf_verifier_env *env,
1599                              const struct btf_type *t)
1600 {
1601         btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct");
1602 }
1603
1604 static struct btf_kind_operations fwd_ops = {
1605         .check_meta = btf_fwd_check_meta,
1606         .resolve = btf_df_resolve,
1607         .check_member = btf_df_check_member,
1608         .check_kflag_member = btf_df_check_kflag_member,
1609         .log_details = btf_fwd_type_log,
1610         .seq_show = btf_df_seq_show,
1611 };
1612
1613 static int btf_array_check_member(struct btf_verifier_env *env,
1614                                   const struct btf_type *struct_type,
1615                                   const struct btf_member *member,
1616                                   const struct btf_type *member_type)
1617 {
1618         u32 struct_bits_off = member->offset;
1619         u32 struct_size, bytes_offset;
1620         u32 array_type_id, array_size;
1621         struct btf *btf = env->btf;
1622
1623         if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1624                 btf_verifier_log_member(env, struct_type, member,
1625                                         "Member is not byte aligned");
1626                 return -EINVAL;
1627         }
1628
1629         array_type_id = member->type;
1630         btf_type_id_size(btf, &array_type_id, &array_size);
1631         struct_size = struct_type->size;
1632         bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1633         if (struct_size - bytes_offset < array_size) {
1634                 btf_verifier_log_member(env, struct_type, member,
1635                                         "Member exceeds struct_size");
1636                 return -EINVAL;
1637         }
1638
1639         return 0;
1640 }
1641
1642 static s32 btf_array_check_meta(struct btf_verifier_env *env,
1643                                 const struct btf_type *t,
1644                                 u32 meta_left)
1645 {
1646         const struct btf_array *array = btf_type_array(t);
1647         u32 meta_needed = sizeof(*array);
1648
1649         if (meta_left < meta_needed) {
1650                 btf_verifier_log_basic(env, t,
1651                                        "meta_left:%u meta_needed:%u",
1652                                        meta_left, meta_needed);
1653                 return -EINVAL;
1654         }
1655
1656         /* array type should not have a name */
1657         if (t->name_off) {
1658                 btf_verifier_log_type(env, t, "Invalid name");
1659                 return -EINVAL;
1660         }
1661
1662         if (btf_type_vlen(t)) {
1663                 btf_verifier_log_type(env, t, "vlen != 0");
1664                 return -EINVAL;
1665         }
1666
1667         if (btf_type_kflag(t)) {
1668                 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
1669                 return -EINVAL;
1670         }
1671
1672         if (t->size) {
1673                 btf_verifier_log_type(env, t, "size != 0");
1674                 return -EINVAL;
1675         }
1676
1677         /* Array elem type and index type cannot be in type void,
1678          * so !array->type and !array->index_type are not allowed.
1679          */
1680         if (!array->type || !BTF_TYPE_ID_VALID(array->type)) {
1681                 btf_verifier_log_type(env, t, "Invalid elem");
1682                 return -EINVAL;
1683         }
1684
1685         if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) {
1686                 btf_verifier_log_type(env, t, "Invalid index");
1687                 return -EINVAL;
1688         }
1689
1690         btf_verifier_log_type(env, t, NULL);
1691
1692         return meta_needed;
1693 }
1694
1695 static int btf_array_resolve(struct btf_verifier_env *env,
1696                              const struct resolve_vertex *v)
1697 {
1698         const struct btf_array *array = btf_type_array(v->t);
1699         const struct btf_type *elem_type, *index_type;
1700         u32 elem_type_id, index_type_id;
1701         struct btf *btf = env->btf;
1702         u32 elem_size;
1703
1704         /* Check array->index_type */
1705         index_type_id = array->index_type;
1706         index_type = btf_type_by_id(btf, index_type_id);
1707         if (btf_type_nosize_or_null(index_type)) {
1708                 btf_verifier_log_type(env, v->t, "Invalid index");
1709                 return -EINVAL;
1710         }
1711
1712         if (!env_type_is_resolve_sink(env, index_type) &&
1713             !env_type_is_resolved(env, index_type_id))
1714                 return env_stack_push(env, index_type, index_type_id);
1715
1716         index_type = btf_type_id_size(btf, &index_type_id, NULL);
1717         if (!index_type || !btf_type_is_int(index_type) ||
1718             !btf_type_int_is_regular(index_type)) {
1719                 btf_verifier_log_type(env, v->t, "Invalid index");
1720                 return -EINVAL;
1721         }
1722
1723         /* Check array->type */
1724         elem_type_id = array->type;
1725         elem_type = btf_type_by_id(btf, elem_type_id);
1726         if (btf_type_nosize_or_null(elem_type)) {
1727                 btf_verifier_log_type(env, v->t,
1728                                       "Invalid elem");
1729                 return -EINVAL;
1730         }
1731
1732         if (!env_type_is_resolve_sink(env, elem_type) &&
1733             !env_type_is_resolved(env, elem_type_id))
1734                 return env_stack_push(env, elem_type, elem_type_id);
1735
1736         elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1737         if (!elem_type) {
1738                 btf_verifier_log_type(env, v->t, "Invalid elem");
1739                 return -EINVAL;
1740         }
1741
1742         if (btf_type_is_int(elem_type) && !btf_type_int_is_regular(elem_type)) {
1743                 btf_verifier_log_type(env, v->t, "Invalid array of int");
1744                 return -EINVAL;
1745         }
1746
1747         if (array->nelems && elem_size > U32_MAX / array->nelems) {
1748                 btf_verifier_log_type(env, v->t,
1749                                       "Array size overflows U32_MAX");
1750                 return -EINVAL;
1751         }
1752
1753         env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems);
1754
1755         return 0;
1756 }
1757
1758 static void btf_array_log(struct btf_verifier_env *env,
1759                           const struct btf_type *t)
1760 {
1761         const struct btf_array *array = btf_type_array(t);
1762
1763         btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u",
1764                          array->type, array->index_type, array->nelems);
1765 }
1766
1767 static void btf_array_seq_show(const struct btf *btf, const struct btf_type *t,
1768                                u32 type_id, void *data, u8 bits_offset,
1769                                struct seq_file *m)
1770 {
1771         const struct btf_array *array = btf_type_array(t);
1772         const struct btf_kind_operations *elem_ops;
1773         const struct btf_type *elem_type;
1774         u32 i, elem_size, elem_type_id;
1775
1776         elem_type_id = array->type;
1777         elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1778         elem_ops = btf_type_ops(elem_type);
1779         seq_puts(m, "[");
1780         for (i = 0; i < array->nelems; i++) {
1781                 if (i)
1782                         seq_puts(m, ",");
1783
1784                 elem_ops->seq_show(btf, elem_type, elem_type_id, data,
1785                                    bits_offset, m);
1786                 data += elem_size;
1787         }
1788         seq_puts(m, "]");
1789 }
1790
1791 static struct btf_kind_operations array_ops = {
1792         .check_meta = btf_array_check_meta,
1793         .resolve = btf_array_resolve,
1794         .check_member = btf_array_check_member,
1795         .check_kflag_member = btf_generic_check_kflag_member,
1796         .log_details = btf_array_log,
1797         .seq_show = btf_array_seq_show,
1798 };
1799
1800 static int btf_struct_check_member(struct btf_verifier_env *env,
1801                                    const struct btf_type *struct_type,
1802                                    const struct btf_member *member,
1803                                    const struct btf_type *member_type)
1804 {
1805         u32 struct_bits_off = member->offset;
1806         u32 struct_size, bytes_offset;
1807
1808         if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1809                 btf_verifier_log_member(env, struct_type, member,
1810                                         "Member is not byte aligned");
1811                 return -EINVAL;
1812         }
1813
1814         struct_size = struct_type->size;
1815         bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1816         if (struct_size - bytes_offset < member_type->size) {
1817                 btf_verifier_log_member(env, struct_type, member,
1818                                         "Member exceeds struct_size");
1819                 return -EINVAL;
1820         }
1821
1822         return 0;
1823 }
1824
1825 static s32 btf_struct_check_meta(struct btf_verifier_env *env,
1826                                  const struct btf_type *t,
1827                                  u32 meta_left)
1828 {
1829         bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION;
1830         const struct btf_member *member;
1831         u32 meta_needed, last_offset;
1832         struct btf *btf = env->btf;
1833         u32 struct_size = t->size;
1834         u32 offset;
1835         u16 i;
1836
1837         meta_needed = btf_type_vlen(t) * sizeof(*member);
1838         if (meta_left < meta_needed) {
1839                 btf_verifier_log_basic(env, t,
1840                                        "meta_left:%u meta_needed:%u",
1841                                        meta_left, meta_needed);
1842                 return -EINVAL;
1843         }
1844
1845         /* struct type either no name or a valid one */
1846         if (t->name_off &&
1847             !btf_name_valid_identifier(env->btf, t->name_off)) {
1848                 btf_verifier_log_type(env, t, "Invalid name");
1849                 return -EINVAL;
1850         }
1851
1852         btf_verifier_log_type(env, t, NULL);
1853
1854         last_offset = 0;
1855         for_each_member(i, t, member) {
1856                 if (!btf_name_offset_valid(btf, member->name_off)) {
1857                         btf_verifier_log_member(env, t, member,
1858                                                 "Invalid member name_offset:%u",
1859                                                 member->name_off);
1860                         return -EINVAL;
1861                 }
1862
1863                 /* struct member either no name or a valid one */
1864                 if (member->name_off &&
1865                     !btf_name_valid_identifier(btf, member->name_off)) {
1866                         btf_verifier_log_member(env, t, member, "Invalid name");
1867                         return -EINVAL;
1868                 }
1869                 /* A member cannot be in type void */
1870                 if (!member->type || !BTF_TYPE_ID_VALID(member->type)) {
1871                         btf_verifier_log_member(env, t, member,
1872                                                 "Invalid type_id");
1873                         return -EINVAL;
1874                 }
1875
1876                 offset = btf_member_bit_offset(t, member);
1877                 if (is_union && offset) {
1878                         btf_verifier_log_member(env, t, member,
1879                                                 "Invalid member bits_offset");
1880                         return -EINVAL;
1881                 }
1882
1883                 /*
1884                  * ">" instead of ">=" because the last member could be
1885                  * "char a[0];"
1886                  */
1887                 if (last_offset > offset) {
1888                         btf_verifier_log_member(env, t, member,
1889                                                 "Invalid member bits_offset");
1890                         return -EINVAL;
1891                 }
1892
1893                 if (BITS_ROUNDUP_BYTES(offset) > struct_size) {
1894                         btf_verifier_log_member(env, t, member,
1895                                                 "Member bits_offset exceeds its struct size");
1896                         return -EINVAL;
1897                 }
1898
1899                 btf_verifier_log_member(env, t, member, NULL);
1900                 last_offset = offset;
1901         }
1902
1903         return meta_needed;
1904 }
1905
1906 static int btf_struct_resolve(struct btf_verifier_env *env,
1907                               const struct resolve_vertex *v)
1908 {
1909         const struct btf_member *member;
1910         int err;
1911         u16 i;
1912
1913         /* Before continue resolving the next_member,
1914          * ensure the last member is indeed resolved to a
1915          * type with size info.
1916          */
1917         if (v->next_member) {
1918                 const struct btf_type *last_member_type;
1919                 const struct btf_member *last_member;
1920                 u16 last_member_type_id;
1921
1922                 last_member = btf_type_member(v->t) + v->next_member - 1;
1923                 last_member_type_id = last_member->type;
1924                 if (WARN_ON_ONCE(!env_type_is_resolved(env,
1925                                                        last_member_type_id)))
1926                         return -EINVAL;
1927
1928                 last_member_type = btf_type_by_id(env->btf,
1929                                                   last_member_type_id);
1930                 if (btf_type_kflag(v->t))
1931                         err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t,
1932                                                                 last_member,
1933                                                                 last_member_type);
1934                 else
1935                         err = btf_type_ops(last_member_type)->check_member(env, v->t,
1936                                                                 last_member,
1937                                                                 last_member_type);
1938                 if (err)
1939                         return err;
1940         }
1941
1942         for_each_member_from(i, v->next_member, v->t, member) {
1943                 u32 member_type_id = member->type;
1944                 const struct btf_type *member_type = btf_type_by_id(env->btf,
1945                                                                 member_type_id);
1946
1947                 if (btf_type_nosize_or_null(member_type)) {
1948                         btf_verifier_log_member(env, v->t, member,
1949                                                 "Invalid member");
1950                         return -EINVAL;
1951                 }
1952
1953                 if (!env_type_is_resolve_sink(env, member_type) &&
1954                     !env_type_is_resolved(env, member_type_id)) {
1955                         env_stack_set_next_member(env, i + 1);
1956                         return env_stack_push(env, member_type, member_type_id);
1957                 }
1958
1959                 if (btf_type_kflag(v->t))
1960                         err = btf_type_ops(member_type)->check_kflag_member(env, v->t,
1961                                                                             member,
1962                                                                             member_type);
1963                 else
1964                         err = btf_type_ops(member_type)->check_member(env, v->t,
1965                                                                       member,
1966                                                                       member_type);
1967                 if (err)
1968                         return err;
1969         }
1970
1971         env_stack_pop_resolved(env, 0, 0);
1972
1973         return 0;
1974 }
1975
1976 static void btf_struct_log(struct btf_verifier_env *env,
1977                            const struct btf_type *t)
1978 {
1979         btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
1980 }
1981
1982 static void btf_struct_seq_show(const struct btf *btf, const struct btf_type *t,
1983                                 u32 type_id, void *data, u8 bits_offset,
1984                                 struct seq_file *m)
1985 {
1986         const char *seq = BTF_INFO_KIND(t->info) == BTF_KIND_UNION ? "|" : ",";
1987         const struct btf_member *member;
1988         u32 i;
1989
1990         seq_puts(m, "{");
1991         for_each_member(i, t, member) {
1992                 const struct btf_type *member_type = btf_type_by_id(btf,
1993                                                                 member->type);
1994                 const struct btf_kind_operations *ops;
1995                 u32 member_offset, bitfield_size;
1996                 u32 bytes_offset;
1997                 u8 bits8_offset;
1998
1999                 if (i)
2000                         seq_puts(m, seq);
2001
2002                 member_offset = btf_member_bit_offset(t, member);
2003                 bitfield_size = btf_member_bitfield_size(t, member);
2004                 bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
2005                 bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
2006                 if (bitfield_size) {
2007                         btf_bitfield_seq_show(data + bytes_offset, bits8_offset,
2008                                               bitfield_size, m);
2009                 } else {
2010                         ops = btf_type_ops(member_type);
2011                         ops->seq_show(btf, member_type, member->type,
2012                                       data + bytes_offset, bits8_offset, m);
2013                 }
2014         }
2015         seq_puts(m, "}");
2016 }
2017
2018 static struct btf_kind_operations struct_ops = {
2019         .check_meta = btf_struct_check_meta,
2020         .resolve = btf_struct_resolve,
2021         .check_member = btf_struct_check_member,
2022         .check_kflag_member = btf_generic_check_kflag_member,
2023         .log_details = btf_struct_log,
2024         .seq_show = btf_struct_seq_show,
2025 };
2026
2027 static int btf_enum_check_member(struct btf_verifier_env *env,
2028                                  const struct btf_type *struct_type,
2029                                  const struct btf_member *member,
2030                                  const struct btf_type *member_type)
2031 {
2032         u32 struct_bits_off = member->offset;
2033         u32 struct_size, bytes_offset;
2034
2035         if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2036                 btf_verifier_log_member(env, struct_type, member,
2037                                         "Member is not byte aligned");
2038                 return -EINVAL;
2039         }
2040
2041         struct_size = struct_type->size;
2042         bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2043         if (struct_size - bytes_offset < sizeof(int)) {
2044                 btf_verifier_log_member(env, struct_type, member,
2045                                         "Member exceeds struct_size");
2046                 return -EINVAL;
2047         }
2048
2049         return 0;
2050 }
2051
2052 static int btf_enum_check_kflag_member(struct btf_verifier_env *env,
2053                                        const struct btf_type *struct_type,
2054                                        const struct btf_member *member,
2055                                        const struct btf_type *member_type)
2056 {
2057         u32 struct_bits_off, nr_bits, bytes_end, struct_size;
2058         u32 int_bitsize = sizeof(int) * BITS_PER_BYTE;
2059
2060         struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset);
2061         nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset);
2062         if (!nr_bits) {
2063                 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2064                         btf_verifier_log_member(env, struct_type, member,
2065                                                 "Member is not byte aligned");
2066                                 return -EINVAL;
2067                 }
2068
2069                 nr_bits = int_bitsize;
2070         } else if (nr_bits > int_bitsize) {
2071                 btf_verifier_log_member(env, struct_type, member,
2072                                         "Invalid member bitfield_size");
2073                 return -EINVAL;
2074         }
2075
2076         struct_size = struct_type->size;
2077         bytes_end = BITS_ROUNDUP_BYTES(struct_bits_off + nr_bits);
2078         if (struct_size < bytes_end) {
2079                 btf_verifier_log_member(env, struct_type, member,
2080                                         "Member exceeds struct_size");
2081                 return -EINVAL;
2082         }
2083
2084         return 0;
2085 }
2086
2087 static s32 btf_enum_check_meta(struct btf_verifier_env *env,
2088                                const struct btf_type *t,
2089                                u32 meta_left)
2090 {
2091         const struct btf_enum *enums = btf_type_enum(t);
2092         struct btf *btf = env->btf;
2093         u16 i, nr_enums;
2094         u32 meta_needed;
2095
2096         nr_enums = btf_type_vlen(t);
2097         meta_needed = nr_enums * sizeof(*enums);
2098
2099         if (meta_left < meta_needed) {
2100                 btf_verifier_log_basic(env, t,
2101                                        "meta_left:%u meta_needed:%u",
2102                                        meta_left, meta_needed);
2103                 return -EINVAL;
2104         }
2105
2106         if (btf_type_kflag(t)) {
2107                 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2108                 return -EINVAL;
2109         }
2110
2111         if (t->size != sizeof(int)) {
2112                 btf_verifier_log_type(env, t, "Expected size:%zu",
2113                                       sizeof(int));
2114                 return -EINVAL;
2115         }
2116
2117         /* enum type either no name or a valid one */
2118         if (t->name_off &&
2119             !btf_name_valid_identifier(env->btf, t->name_off)) {
2120                 btf_verifier_log_type(env, t, "Invalid name");
2121                 return -EINVAL;
2122         }
2123
2124         btf_verifier_log_type(env, t, NULL);
2125
2126         for (i = 0; i < nr_enums; i++) {
2127                 if (!btf_name_offset_valid(btf, enums[i].name_off)) {
2128                         btf_verifier_log(env, "\tInvalid name_offset:%u",
2129                                          enums[i].name_off);
2130                         return -EINVAL;
2131                 }
2132
2133                 /* enum member must have a valid name */
2134                 if (!enums[i].name_off ||
2135                     !btf_name_valid_identifier(btf, enums[i].name_off)) {
2136                         btf_verifier_log_type(env, t, "Invalid name");
2137                         return -EINVAL;
2138                 }
2139
2140
2141                 btf_verifier_log(env, "\t%s val=%d\n",
2142                                  __btf_name_by_offset(btf, enums[i].name_off),
2143                                  enums[i].val);
2144         }
2145
2146         return meta_needed;
2147 }
2148
2149 static void btf_enum_log(struct btf_verifier_env *env,
2150                          const struct btf_type *t)
2151 {
2152         btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
2153 }
2154
2155 static void btf_enum_seq_show(const struct btf *btf, const struct btf_type *t,
2156                               u32 type_id, void *data, u8 bits_offset,
2157                               struct seq_file *m)
2158 {
2159         const struct btf_enum *enums = btf_type_enum(t);
2160         u32 i, nr_enums = btf_type_vlen(t);
2161         int v = *(int *)data;
2162
2163         for (i = 0; i < nr_enums; i++) {
2164                 if (v == enums[i].val) {
2165                         seq_printf(m, "%s",
2166                                    __btf_name_by_offset(btf,
2167                                                         enums[i].name_off));
2168                         return;
2169                 }
2170         }
2171
2172         seq_printf(m, "%d", v);
2173 }
2174
2175 static struct btf_kind_operations enum_ops = {
2176         .check_meta = btf_enum_check_meta,
2177         .resolve = btf_df_resolve,
2178         .check_member = btf_enum_check_member,
2179         .check_kflag_member = btf_enum_check_kflag_member,
2180         .log_details = btf_enum_log,
2181         .seq_show = btf_enum_seq_show,
2182 };
2183
2184 static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,
2185                                      const struct btf_type *t,
2186                                      u32 meta_left)
2187 {
2188         u32 meta_needed = btf_type_vlen(t) * sizeof(struct btf_param);
2189
2190         if (meta_left < meta_needed) {
2191                 btf_verifier_log_basic(env, t,
2192                                        "meta_left:%u meta_needed:%u",
2193                                        meta_left, meta_needed);
2194                 return -EINVAL;
2195         }
2196
2197         if (t->name_off) {
2198                 btf_verifier_log_type(env, t, "Invalid name");
2199                 return -EINVAL;
2200         }
2201
2202         if (btf_type_kflag(t)) {
2203                 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2204                 return -EINVAL;
2205         }
2206
2207         btf_verifier_log_type(env, t, NULL);
2208
2209         return meta_needed;
2210 }
2211
2212 static void btf_func_proto_log(struct btf_verifier_env *env,
2213                                const struct btf_type *t)
2214 {
2215         const struct btf_param *args = (const struct btf_param *)(t + 1);
2216         u16 nr_args = btf_type_vlen(t), i;
2217
2218         btf_verifier_log(env, "return=%u args=(", t->type);
2219         if (!nr_args) {
2220                 btf_verifier_log(env, "void");
2221                 goto done;
2222         }
2223
2224         if (nr_args == 1 && !args[0].type) {
2225                 /* Only one vararg */
2226                 btf_verifier_log(env, "vararg");
2227                 goto done;
2228         }
2229
2230         btf_verifier_log(env, "%u %s", args[0].type,
2231                          __btf_name_by_offset(env->btf,
2232                                               args[0].name_off));
2233         for (i = 1; i < nr_args - 1; i++)
2234                 btf_verifier_log(env, ", %u %s", args[i].type,
2235                                  __btf_name_by_offset(env->btf,
2236                                                       args[i].name_off));
2237
2238         if (nr_args > 1) {
2239                 const struct btf_param *last_arg = &args[nr_args - 1];
2240
2241                 if (last_arg->type)
2242                         btf_verifier_log(env, ", %u %s", last_arg->type,
2243                                          __btf_name_by_offset(env->btf,
2244                                                               last_arg->name_off));
2245                 else
2246                         btf_verifier_log(env, ", vararg");
2247         }
2248
2249 done:
2250         btf_verifier_log(env, ")");
2251 }
2252
2253 static struct btf_kind_operations func_proto_ops = {
2254         .check_meta = btf_func_proto_check_meta,
2255         .resolve = btf_df_resolve,
2256         /*
2257          * BTF_KIND_FUNC_PROTO cannot be directly referred by
2258          * a struct's member.
2259          *
2260          * It should be a funciton pointer instead.
2261          * (i.e. struct's member -> BTF_KIND_PTR -> BTF_KIND_FUNC_PROTO)
2262          *
2263          * Hence, there is no btf_func_check_member().
2264          */
2265         .check_member = btf_df_check_member,
2266         .check_kflag_member = btf_df_check_kflag_member,
2267         .log_details = btf_func_proto_log,
2268         .seq_show = btf_df_seq_show,
2269 };
2270
2271 static s32 btf_func_check_meta(struct btf_verifier_env *env,
2272                                const struct btf_type *t,
2273                                u32 meta_left)
2274 {
2275         if (!t->name_off ||
2276             !btf_name_valid_identifier(env->btf, t->name_off)) {
2277                 btf_verifier_log_type(env, t, "Invalid name");
2278                 return -EINVAL;
2279         }
2280
2281         if (btf_type_vlen(t)) {
2282                 btf_verifier_log_type(env, t, "vlen != 0");
2283                 return -EINVAL;
2284         }
2285
2286         if (btf_type_kflag(t)) {
2287                 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2288                 return -EINVAL;
2289         }
2290
2291         btf_verifier_log_type(env, t, NULL);
2292
2293         return 0;
2294 }
2295
2296 static struct btf_kind_operations func_ops = {
2297         .check_meta = btf_func_check_meta,
2298         .resolve = btf_df_resolve,
2299         .check_member = btf_df_check_member,
2300         .check_kflag_member = btf_df_check_kflag_member,
2301         .log_details = btf_ref_type_log,
2302         .seq_show = btf_df_seq_show,
2303 };
2304
2305 static int btf_func_proto_check(struct btf_verifier_env *env,
2306                                 const struct btf_type *t)
2307 {
2308         const struct btf_type *ret_type;
2309         const struct btf_param *args;
2310         const struct btf *btf;
2311         u16 nr_args, i;
2312         int err;
2313
2314         btf = env->btf;
2315         args = (const struct btf_param *)(t + 1);
2316         nr_args = btf_type_vlen(t);
2317
2318         /* Check func return type which could be "void" (t->type == 0) */
2319         if (t->type) {
2320                 u32 ret_type_id = t->type;
2321
2322                 ret_type = btf_type_by_id(btf, ret_type_id);
2323                 if (!ret_type) {
2324                         btf_verifier_log_type(env, t, "Invalid return type");
2325                         return -EINVAL;
2326                 }
2327
2328                 if (btf_type_needs_resolve(ret_type) &&
2329                     !env_type_is_resolved(env, ret_type_id)) {
2330                         err = btf_resolve(env, ret_type, ret_type_id);
2331                         if (err)
2332                                 return err;
2333                 }
2334
2335                 /* Ensure the return type is a type that has a size */
2336                 if (!btf_type_id_size(btf, &ret_type_id, NULL)) {
2337                         btf_verifier_log_type(env, t, "Invalid return type");
2338                         return -EINVAL;
2339                 }
2340         }
2341
2342         if (!nr_args)
2343                 return 0;
2344
2345         /* Last func arg type_id could be 0 if it is a vararg */
2346         if (!args[nr_args - 1].type) {
2347                 if (args[nr_args - 1].name_off) {
2348                         btf_verifier_log_type(env, t, "Invalid arg#%u",
2349                                               nr_args);
2350                         return -EINVAL;
2351                 }
2352                 nr_args--;
2353         }
2354
2355         err = 0;
2356         for (i = 0; i < nr_args; i++) {
2357                 const struct btf_type *arg_type;
2358                 u32 arg_type_id;
2359
2360                 arg_type_id = args[i].type;
2361                 arg_type = btf_type_by_id(btf, arg_type_id);
2362                 if (!arg_type) {
2363                         btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
2364                         err = -EINVAL;
2365                         break;
2366                 }
2367
2368                 if (args[i].name_off &&
2369                     (!btf_name_offset_valid(btf, args[i].name_off) ||
2370                      !btf_name_valid_identifier(btf, args[i].name_off))) {
2371                         btf_verifier_log_type(env, t,
2372                                               "Invalid arg#%u", i + 1);
2373                         err = -EINVAL;
2374                         break;
2375                 }
2376
2377                 if (btf_type_needs_resolve(arg_type) &&
2378                     !env_type_is_resolved(env, arg_type_id)) {
2379                         err = btf_resolve(env, arg_type, arg_type_id);
2380                         if (err)
2381                                 break;
2382                 }
2383
2384                 if (!btf_type_id_size(btf, &arg_type_id, NULL)) {
2385                         btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
2386                         err = -EINVAL;
2387                         break;
2388                 }
2389         }
2390
2391         return err;
2392 }
2393
2394 static int btf_func_check(struct btf_verifier_env *env,
2395                           const struct btf_type *t)
2396 {
2397         const struct btf_type *proto_type;
2398         const struct btf_param *args;
2399         const struct btf *btf;
2400         u16 nr_args, i;
2401
2402         btf = env->btf;
2403         proto_type = btf_type_by_id(btf, t->type);
2404
2405         if (!proto_type || !btf_type_is_func_proto(proto_type)) {
2406                 btf_verifier_log_type(env, t, "Invalid type_id");
2407                 return -EINVAL;
2408         }
2409
2410         args = (const struct btf_param *)(proto_type + 1);
2411         nr_args = btf_type_vlen(proto_type);
2412         for (i = 0; i < nr_args; i++) {
2413                 if (!args[i].name_off && args[i].type) {
2414                         btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
2415                         return -EINVAL;
2416                 }
2417         }
2418
2419         return 0;
2420 }
2421
2422 static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = {
2423         [BTF_KIND_INT] = &int_ops,
2424         [BTF_KIND_PTR] = &ptr_ops,
2425         [BTF_KIND_ARRAY] = &array_ops,
2426         [BTF_KIND_STRUCT] = &struct_ops,
2427         [BTF_KIND_UNION] = &struct_ops,
2428         [BTF_KIND_ENUM] = &enum_ops,
2429         [BTF_KIND_FWD] = &fwd_ops,
2430         [BTF_KIND_TYPEDEF] = &modifier_ops,
2431         [BTF_KIND_VOLATILE] = &modifier_ops,
2432         [BTF_KIND_CONST] = &modifier_ops,
2433         [BTF_KIND_RESTRICT] = &modifier_ops,
2434         [BTF_KIND_FUNC] = &func_ops,
2435         [BTF_KIND_FUNC_PROTO] = &func_proto_ops,
2436 };
2437
2438 static s32 btf_check_meta(struct btf_verifier_env *env,
2439                           const struct btf_type *t,
2440                           u32 meta_left)
2441 {
2442         u32 saved_meta_left = meta_left;
2443         s32 var_meta_size;
2444
2445         if (meta_left < sizeof(*t)) {
2446                 btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu",
2447                                  env->log_type_id, meta_left, sizeof(*t));
2448                 return -EINVAL;
2449         }
2450         meta_left -= sizeof(*t);
2451
2452         if (t->info & ~BTF_INFO_MASK) {
2453                 btf_verifier_log(env, "[%u] Invalid btf_info:%x",
2454                                  env->log_type_id, t->info);
2455                 return -EINVAL;
2456         }
2457
2458         if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX ||
2459             BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) {
2460                 btf_verifier_log(env, "[%u] Invalid kind:%u",
2461                                  env->log_type_id, BTF_INFO_KIND(t->info));
2462                 return -EINVAL;
2463         }
2464
2465         if (!btf_name_offset_valid(env->btf, t->name_off)) {
2466                 btf_verifier_log(env, "[%u] Invalid name_offset:%u",
2467                                  env->log_type_id, t->name_off);
2468                 return -EINVAL;
2469         }
2470
2471         var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left);
2472         if (var_meta_size < 0)
2473                 return var_meta_size;
2474
2475         meta_left -= var_meta_size;
2476
2477         return saved_meta_left - meta_left;
2478 }
2479
2480 static int btf_check_all_metas(struct btf_verifier_env *env)
2481 {
2482         struct btf *btf = env->btf;
2483         struct btf_header *hdr;
2484         void *cur, *end;
2485
2486         hdr = &btf->hdr;
2487         cur = btf->nohdr_data + hdr->type_off;
2488         end = cur + hdr->type_len;
2489
2490         env->log_type_id = 1;
2491         while (cur < end) {
2492                 struct btf_type *t = cur;
2493                 s32 meta_size;
2494
2495                 meta_size = btf_check_meta(env, t, end - cur);
2496                 if (meta_size < 0)
2497                         return meta_size;
2498
2499                 btf_add_type(env, t);
2500                 cur += meta_size;
2501                 env->log_type_id++;
2502         }
2503
2504         return 0;
2505 }
2506
2507 static bool btf_resolve_valid(struct btf_verifier_env *env,
2508                               const struct btf_type *t,
2509                               u32 type_id)
2510 {
2511         struct btf *btf = env->btf;
2512
2513         if (!env_type_is_resolved(env, type_id))
2514                 return false;
2515
2516         if (btf_type_is_struct(t))
2517                 return !btf->resolved_ids[type_id] &&
2518                         !btf->resolved_sizes[type_id];
2519
2520         if (btf_type_is_modifier(t) || btf_type_is_ptr(t)) {
2521                 t = btf_type_id_resolve(btf, &type_id);
2522                 return t && !btf_type_is_modifier(t);
2523         }
2524
2525         if (btf_type_is_array(t)) {
2526                 const struct btf_array *array = btf_type_array(t);
2527                 const struct btf_type *elem_type;
2528                 u32 elem_type_id = array->type;
2529                 u32 elem_size;
2530
2531                 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
2532                 return elem_type && !btf_type_is_modifier(elem_type) &&
2533                         (array->nelems * elem_size ==
2534                          btf->resolved_sizes[type_id]);
2535         }
2536
2537         return false;
2538 }
2539
2540 static int btf_resolve(struct btf_verifier_env *env,
2541                        const struct btf_type *t, u32 type_id)
2542 {
2543         u32 save_log_type_id = env->log_type_id;
2544         const struct resolve_vertex *v;
2545         int err = 0;
2546
2547         env->resolve_mode = RESOLVE_TBD;
2548         env_stack_push(env, t, type_id);
2549         while (!err && (v = env_stack_peak(env))) {
2550                 env->log_type_id = v->type_id;
2551                 err = btf_type_ops(v->t)->resolve(env, v);
2552         }
2553
2554         env->log_type_id = type_id;
2555         if (err == -E2BIG) {
2556                 btf_verifier_log_type(env, t,
2557                                       "Exceeded max resolving depth:%u",
2558                                       MAX_RESOLVE_DEPTH);
2559         } else if (err == -EEXIST) {
2560                 btf_verifier_log_type(env, t, "Loop detected");
2561         }
2562
2563         /* Final sanity check */
2564         if (!err && !btf_resolve_valid(env, t, type_id)) {
2565                 btf_verifier_log_type(env, t, "Invalid resolve state");
2566                 err = -EINVAL;
2567         }
2568
2569         env->log_type_id = save_log_type_id;
2570         return err;
2571 }
2572
2573 static int btf_check_all_types(struct btf_verifier_env *env)
2574 {
2575         struct btf *btf = env->btf;
2576         u32 type_id;
2577         int err;
2578
2579         err = env_resolve_init(env);
2580         if (err)
2581                 return err;
2582
2583         env->phase++;
2584         for (type_id = 1; type_id <= btf->nr_types; type_id++) {
2585                 const struct btf_type *t = btf_type_by_id(btf, type_id);
2586
2587                 env->log_type_id = type_id;
2588                 if (btf_type_needs_resolve(t) &&
2589                     !env_type_is_resolved(env, type_id)) {
2590                         err = btf_resolve(env, t, type_id);
2591                         if (err)
2592                                 return err;
2593                 }
2594
2595                 if (btf_type_is_func_proto(t)) {
2596                         err = btf_func_proto_check(env, t);
2597                         if (err)
2598                                 return err;
2599                 }
2600
2601                 if (btf_type_is_func(t)) {
2602                         err = btf_func_check(env, t);
2603                         if (err)
2604                                 return err;
2605                 }
2606         }
2607
2608         return 0;
2609 }
2610
2611 static int btf_parse_type_sec(struct btf_verifier_env *env)
2612 {
2613         const struct btf_header *hdr = &env->btf->hdr;
2614         int err;
2615
2616         /* Type section must align to 4 bytes */
2617         if (hdr->type_off & (sizeof(u32) - 1)) {
2618                 btf_verifier_log(env, "Unaligned type_off");
2619                 return -EINVAL;
2620         }
2621
2622         if (!hdr->type_len) {
2623                 btf_verifier_log(env, "No type found");
2624                 return -EINVAL;
2625         }
2626
2627         err = btf_check_all_metas(env);
2628         if (err)
2629                 return err;
2630
2631         return btf_check_all_types(env);
2632 }
2633
2634 static int btf_parse_str_sec(struct btf_verifier_env *env)
2635 {
2636         const struct btf_header *hdr;
2637         struct btf *btf = env->btf;
2638         const char *start, *end;
2639
2640         hdr = &btf->hdr;
2641         start = btf->nohdr_data + hdr->str_off;
2642         end = start + hdr->str_len;
2643
2644         if (end != btf->data + btf->data_size) {
2645                 btf_verifier_log(env, "String section is not at the end");
2646                 return -EINVAL;
2647         }
2648
2649         if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET ||
2650             start[0] || end[-1]) {
2651                 btf_verifier_log(env, "Invalid string section");
2652                 return -EINVAL;
2653         }
2654
2655         btf->strings = start;
2656
2657         return 0;
2658 }
2659
2660 static const size_t btf_sec_info_offset[] = {
2661         offsetof(struct btf_header, type_off),
2662         offsetof(struct btf_header, str_off),
2663 };
2664
2665 static int btf_sec_info_cmp(const void *a, const void *b)
2666 {
2667         const struct btf_sec_info *x = a;
2668         const struct btf_sec_info *y = b;
2669
2670         return (int)(x->off - y->off) ? : (int)(x->len - y->len);
2671 }
2672
2673 static int btf_check_sec_info(struct btf_verifier_env *env,
2674                               u32 btf_data_size)
2675 {
2676         struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)];
2677         u32 total, expected_total, i;
2678         const struct btf_header *hdr;
2679         const struct btf *btf;
2680
2681         btf = env->btf;
2682         hdr = &btf->hdr;
2683
2684         /* Populate the secs from hdr */
2685         for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++)
2686                 secs[i] = *(struct btf_sec_info *)((void *)hdr +
2687                                                    btf_sec_info_offset[i]);
2688
2689         sort(secs, ARRAY_SIZE(btf_sec_info_offset),
2690              sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL);
2691
2692         /* Check for gaps and overlap among sections */
2693         total = 0;
2694         expected_total = btf_data_size - hdr->hdr_len;
2695         for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) {
2696                 if (expected_total < secs[i].off) {
2697                         btf_verifier_log(env, "Invalid section offset");
2698                         return -EINVAL;
2699                 }
2700                 if (total < secs[i].off) {
2701                         /* gap */
2702                         btf_verifier_log(env, "Unsupported section found");
2703                         return -EINVAL;
2704                 }
2705                 if (total > secs[i].off) {
2706                         btf_verifier_log(env, "Section overlap found");
2707                         return -EINVAL;
2708                 }
2709                 if (expected_total - total < secs[i].len) {
2710                         btf_verifier_log(env,
2711                                          "Total section length too long");
2712                         return -EINVAL;
2713                 }
2714                 total += secs[i].len;
2715         }
2716
2717         /* There is data other than hdr and known sections */
2718         if (expected_total != total) {
2719                 btf_verifier_log(env, "Unsupported section found");
2720                 return -EINVAL;
2721         }
2722
2723         return 0;
2724 }
2725
2726 static int btf_parse_hdr(struct btf_verifier_env *env)
2727 {
2728         u32 hdr_len, hdr_copy, btf_data_size;
2729         const struct btf_header *hdr;
2730         struct btf *btf;
2731         int err;
2732
2733         btf = env->btf;
2734         btf_data_size = btf->data_size;
2735
2736         if (btf_data_size <
2737             offsetof(struct btf_header, hdr_len) + sizeof(hdr->hdr_len)) {
2738                 btf_verifier_log(env, "hdr_len not found");
2739                 return -EINVAL;
2740         }
2741
2742         hdr = btf->data;
2743         hdr_len = hdr->hdr_len;
2744         if (btf_data_size < hdr_len) {
2745                 btf_verifier_log(env, "btf_header not found");
2746                 return -EINVAL;
2747         }
2748
2749         /* Ensure the unsupported header fields are zero */
2750         if (hdr_len > sizeof(btf->hdr)) {
2751                 u8 *expected_zero = btf->data + sizeof(btf->hdr);
2752                 u8 *end = btf->data + hdr_len;
2753
2754                 for (; expected_zero < end; expected_zero++) {
2755                         if (*expected_zero) {
2756                                 btf_verifier_log(env, "Unsupported btf_header");
2757                                 return -E2BIG;
2758                         }
2759                 }
2760         }
2761
2762         hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr));
2763         memcpy(&btf->hdr, btf->data, hdr_copy);
2764
2765         hdr = &btf->hdr;
2766
2767         btf_verifier_log_hdr(env, btf_data_size);
2768
2769         if (hdr->magic != BTF_MAGIC) {
2770                 btf_verifier_log(env, "Invalid magic");
2771                 return -EINVAL;
2772         }
2773
2774         if (hdr->version != BTF_VERSION) {
2775                 btf_verifier_log(env, "Unsupported version");
2776                 return -ENOTSUPP;
2777         }
2778
2779         if (hdr->flags) {
2780                 btf_verifier_log(env, "Unsupported flags");
2781                 return -ENOTSUPP;
2782         }
2783
2784         if (btf_data_size == hdr->hdr_len) {
2785                 btf_verifier_log(env, "No data");
2786                 return -EINVAL;
2787         }
2788
2789         err = btf_check_sec_info(env, btf_data_size);
2790         if (err)
2791                 return err;
2792
2793         return 0;
2794 }
2795
2796 static struct btf *btf_parse(void __user *btf_data, u32 btf_data_size,
2797                              u32 log_level, char __user *log_ubuf, u32 log_size)
2798 {
2799         struct btf_verifier_env *env = NULL;
2800         struct bpf_verifier_log *log;
2801         struct btf *btf = NULL;
2802         u8 *data;
2803         int err;
2804
2805         if (btf_data_size > BTF_MAX_SIZE)
2806                 return ERR_PTR(-E2BIG);
2807
2808         env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
2809         if (!env)
2810                 return ERR_PTR(-ENOMEM);
2811
2812         log = &env->log;
2813         if (log_level || log_ubuf || log_size) {
2814                 /* user requested verbose verifier output
2815                  * and supplied buffer to store the verification trace
2816                  */
2817                 log->level = log_level;
2818                 log->ubuf = log_ubuf;
2819                 log->len_total = log_size;
2820
2821                 /* log attributes have to be sane */
2822                 if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 ||
2823                     !log->level || !log->ubuf) {
2824                         err = -EINVAL;
2825                         goto errout;
2826                 }
2827         }
2828
2829         btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
2830         if (!btf) {
2831                 err = -ENOMEM;
2832                 goto errout;
2833         }
2834         env->btf = btf;
2835
2836         data = kvmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN);
2837         if (!data) {
2838                 err = -ENOMEM;
2839                 goto errout;
2840         }
2841
2842         btf->data = data;
2843         btf->data_size = btf_data_size;
2844
2845         if (copy_from_user(data, btf_data, btf_data_size)) {
2846                 err = -EFAULT;
2847                 goto errout;
2848         }
2849
2850         err = btf_parse_hdr(env);
2851         if (err)
2852                 goto errout;
2853
2854         btf->nohdr_data = btf->data + btf->hdr.hdr_len;
2855
2856         err = btf_parse_str_sec(env);
2857         if (err)
2858                 goto errout;
2859
2860         err = btf_parse_type_sec(env);
2861         if (err)
2862                 goto errout;
2863
2864         if (log->level && bpf_verifier_log_full(log)) {
2865                 err = -ENOSPC;
2866                 goto errout;
2867         }
2868
2869         btf_verifier_env_free(env);
2870         refcount_set(&btf->refcnt, 1);
2871         return btf;
2872
2873 errout:
2874         btf_verifier_env_free(env);
2875         if (btf)
2876                 btf_free(btf);
2877         return ERR_PTR(err);
2878 }
2879
2880 void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
2881                        struct seq_file *m)
2882 {
2883         const struct btf_type *t = btf_type_by_id(btf, type_id);
2884
2885         btf_type_ops(t)->seq_show(btf, t, type_id, obj, 0, m);
2886 }
2887
2888 static int btf_release(struct inode *inode, struct file *filp)
2889 {
2890         btf_put(filp->private_data);
2891         return 0;
2892 }
2893
2894 const struct file_operations btf_fops = {
2895         .release        = btf_release,
2896 };
2897
2898 static int __btf_new_fd(struct btf *btf)
2899 {
2900         return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC);
2901 }
2902
2903 int btf_new_fd(const union bpf_attr *attr)
2904 {
2905         struct btf *btf;
2906         int ret;
2907
2908         btf = btf_parse(u64_to_user_ptr(attr->btf),
2909                         attr->btf_size, attr->btf_log_level,
2910                         u64_to_user_ptr(attr->btf_log_buf),
2911                         attr->btf_log_size);
2912         if (IS_ERR(btf))
2913                 return PTR_ERR(btf);
2914
2915         ret = btf_alloc_id(btf);
2916         if (ret) {
2917                 btf_free(btf);
2918                 return ret;
2919         }
2920
2921         /*
2922          * The BTF ID is published to the userspace.
2923          * All BTF free must go through call_rcu() from
2924          * now on (i.e. free by calling btf_put()).
2925          */
2926
2927         ret = __btf_new_fd(btf);
2928         if (ret < 0)
2929                 btf_put(btf);
2930
2931         return ret;
2932 }
2933
2934 struct btf *btf_get_by_fd(int fd)
2935 {
2936         struct btf *btf;
2937         struct fd f;
2938
2939         f = fdget(fd);
2940
2941         if (!f.file)
2942                 return ERR_PTR(-EBADF);
2943
2944         if (f.file->f_op != &btf_fops) {
2945                 fdput(f);
2946                 return ERR_PTR(-EINVAL);
2947         }
2948
2949         btf = f.file->private_data;
2950         refcount_inc(&btf->refcnt);
2951         fdput(f);
2952
2953         return btf;
2954 }
2955
2956 int btf_get_info_by_fd(const struct btf *btf,
2957                        const union bpf_attr *attr,
2958                        union bpf_attr __user *uattr)
2959 {
2960         struct bpf_btf_info __user *uinfo;
2961         struct bpf_btf_info info = {};
2962         u32 info_copy, btf_copy;
2963         void __user *ubtf;
2964         u32 uinfo_len;
2965
2966         uinfo = u64_to_user_ptr(attr->info.info);
2967         uinfo_len = attr->info.info_len;
2968
2969         info_copy = min_t(u32, uinfo_len, sizeof(info));
2970         if (copy_from_user(&info, uinfo, info_copy))
2971                 return -EFAULT;
2972
2973         info.id = btf->id;
2974         ubtf = u64_to_user_ptr(info.btf);
2975         btf_copy = min_t(u32, btf->data_size, info.btf_size);
2976         if (copy_to_user(ubtf, btf->data, btf_copy))
2977                 return -EFAULT;
2978         info.btf_size = btf->data_size;
2979
2980         if (copy_to_user(uinfo, &info, info_copy) ||
2981             put_user(info_copy, &uattr->info.info_len))
2982                 return -EFAULT;
2983
2984         return 0;
2985 }
2986
2987 int btf_get_fd_by_id(u32 id)
2988 {
2989         struct btf *btf;
2990         int fd;
2991
2992         rcu_read_lock();
2993         btf = idr_find(&btf_idr, id);
2994         if (!btf || !refcount_inc_not_zero(&btf->refcnt))
2995                 btf = ERR_PTR(-ENOENT);
2996         rcu_read_unlock();
2997
2998         if (IS_ERR(btf))
2999                 return PTR_ERR(btf);
3000
3001         fd = __btf_new_fd(btf);
3002         if (fd < 0)
3003                 btf_put(btf);
3004
3005         return fd;
3006 }
3007
3008 u32 btf_id(const struct btf *btf)
3009 {
3010         return btf->id;
3011 }