Unlike x86-64, arm64 places alternative replacement instructions in .text, immediately after the affected function.
So if the replacement instructions have PC-relative branches without relocations, their offsets relative to the function have to remain constant. Achieve that by cloning the function's alternative replacements immediately after cloning the function itself. Signed-off-by: Josh Poimboeuf <[email protected]> --- tools/objtool/elf.c | 9 ++--- tools/objtool/include/objtool/elf.h | 6 +++- tools/objtool/klp-diff.c | 54 ++++++++++++++++++++++++----- tools/objtool/klp-symid.c | 2 +- 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index f9d49325dbe0d..f6219f495efdc 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -1426,14 +1426,15 @@ int elf_add_string(struct elf *elf, struct section *strtab, const char *str) return -1; } - data = elf_add_data(elf, strtab, str, strlen(str) + 1); + data = elf_add_data(elf, strtab, str, strlen(str) + 1, true); if (!data) return -1; return data - strtab->data->d_buf; } -void *elf_add_data(struct elf *elf, struct section *sec, const void *data, size_t size) +void *elf_add_data(struct elf *elf, struct section *sec, const void *data, + size_t size, bool align) { unsigned long offset, size_old, size_new, alloc_size_old, alloc_size_new; Elf_Scn *s; @@ -1460,7 +1461,7 @@ void *elf_add_data(struct elf *elf, struct section *sec, const void *data, size_ } size_old = sec->data->d_size; - offset = ALIGN(size_old, sec->sh.sh_addralign); + offset = ALIGN(size_old, align ? sec->sh.sh_addralign : 1); size_new = offset + size; if (!sec->data_overallocated) @@ -1603,7 +1604,7 @@ static int elf_alloc_reloc(struct elf *elf, struct section *rsec) unsigned long nr_alloc_old = 0, nr_alloc_new; struct symbol *sym; - if (!elf_add_data(elf, rsec, NULL, elf_rela_size(elf))) + if (!elf_add_data(elf, rsec, NULL, elf_rela_size(elf), true)) return -1; rsec->data->d_type = ELF_T_RELA; diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index d33d25d7472a0..fba0a0e08f8b6 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -107,6 +107,7 @@ struct symbol { u8 klp : 1; u8 dont_correlate : 1; u8 fake : 1; + u8 unalign : 1; struct list_head pv_target; struct reloc *relocs; struct section *group_sec; @@ -187,7 +188,7 @@ struct symbol *elf_create_symbol(struct elf *elf, const char *name, struct symbol *elf_create_section_symbol(struct elf *elf, struct section *sec); void *elf_add_data(struct elf *elf, struct section *sec, const void *data, - size_t size); + size_t size, bool align); int elf_find_string(struct elf *elf, struct section *strtab, const char *str); int elf_add_string(struct elf *elf, struct section *strtab, const char *str); @@ -533,6 +534,9 @@ static inline void set_sym_next_reloc(struct reloc *reloc, struct reloc *next) #define sec_for_each_sym_from(sec, sym) \ list_for_each_entry_from(sym, &sec->symbol_list, list) +#define sec_for_each_sym_continue(sec, sym) \ + list_for_each_entry_continue(sym, &sec->symbol_list, list) + #define sec_prev_sym(sym) \ sym->sec && sym->list.prev != &sym->sec->symbol_list ? \ list_prev_entry(sym, list) : NULL diff --git a/tools/objtool/klp-diff.c b/tools/objtool/klp-diff.c index d76d2f5b5acbf..841651e3ad746 100644 --- a/tools/objtool/klp-diff.c +++ b/tools/objtool/klp-diff.c @@ -1014,8 +1014,9 @@ static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym); static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym, bool data_too) { - struct section *out_sec = NULL; unsigned long offset = 0, pfx_size = 0; + bool align = !patched_sym->unalign; + struct section *out_sec = NULL; struct symbol *out_sym; if (data_too && !is_undef_sym(patched_sym)) { @@ -1041,7 +1042,7 @@ static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym } if (!is_sec_sym(patched_sym)) - offset = ALIGN(sec_size(out_sec), out_sec->sh.sh_addralign); + offset = ALIGN(sec_size(out_sec), align ? out_sec->sh.sh_addralign : 1); if (patched_sym->len || is_sec_sym(patched_sym)) { void *data = NULL; @@ -1059,7 +1060,7 @@ static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym else size = patched_sym->len + pfx_size; - if (!elf_add_data(elf, out_sec, data, size)) + if (!elf_add_data(elf, out_sec, data, size, align)) return NULL; offset += pfx_size; @@ -1101,6 +1102,37 @@ static const char *sym_bind(struct symbol *sym) } } +static struct symbol *clone_symbol(struct elfs *e, struct symbol *patched_sym, + bool data_too); + +/* + * For arm64 alternatives, the replacement instructions come immediately after + * the function. Clone any such blocks of instructions in place to preserve + * their offsets relative to the function in case they have hard-coded PC + * relative branches. + */ +static int clone_inline_alternatives(struct elfs *e, struct symbol *patched_sym) +{ + struct symbol *next; + + if (!__is_defined(ARCH_HAS_INLINE_ALTS) || !is_func_sym(patched_sym)) + return 0; + + next = patched_sym; + sec_for_each_sym_continue(patched_sym->sec, next) { + if (next->offset < (patched_sym->offset + patched_sym->len) || + is_mapping_sym(next)) + continue; + if (!next->fake) + break; + next->unalign = 1; + if (!clone_symbol(e, next, true)) + return -1; + } + + return 0; +} + /* * Copy a symbol to the output object, optionally including its data and * relocations. @@ -1125,7 +1157,13 @@ static struct symbol *clone_symbol(struct elfs *e, struct symbol *patched_sym, if (!__clone_symbol(e->out, patched_sym, data_too)) return NULL; - if (data_too && clone_sym_relocs(e, patched_sym)) + if (!data_too || is_undef_sym(patched_sym)) + return patched_sym->clone; + + if (clone_sym_relocs(e, patched_sym)) + return NULL; + + if (clone_inline_alternatives(e, patched_sym)) return NULL; return patched_sym->clone; @@ -1585,7 +1623,7 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, memset(&klp_reloc, 0, sizeof(klp_reloc)); klp_reloc.type = reloc_type(patched_reloc); - if (!elf_add_data(e->out, klp_relocs, &klp_reloc, sizeof(klp_reloc))) + if (!elf_add_data(e->out, klp_relocs, &klp_reloc, sizeof(klp_reloc), true)) return -1; /* klp_reloc.offset */ @@ -2164,7 +2202,7 @@ static int create_klp_sections(struct elfs *e) return -1; /* allocate klp_object_ext */ - obj_data = elf_add_data(e->out, obj_sec, NULL, obj_size); + obj_data = elf_add_data(e->out, obj_sec, NULL, obj_size, true); if (!obj_data) return -1; @@ -2199,7 +2237,7 @@ static int create_klp_sections(struct elfs *e) continue; /* allocate klp_func_ext */ - func_data = elf_add_data(e->out, funcs_sec, NULL, func_size); + func_data = elf_add_data(e->out, funcs_sec, NULL, func_size, true); if (!func_data) return -1; @@ -2345,7 +2383,7 @@ static int copy_import_ns(struct elfs *e) } } - if (!elf_add_data(e->out, out_sec, import_ns, strlen(import_ns) + 1)) + if (!elf_add_data(e->out, out_sec, import_ns, strlen(import_ns) + 1, true)) return -1; } diff --git a/tools/objtool/klp-symid.c b/tools/objtool/klp-symid.c index 21d8708013aba..1934c2a1afc4d 100644 --- a/tools/objtool/klp-symid.c +++ b/tools/objtool/klp-symid.c @@ -95,7 +95,7 @@ int klp_create_symid_sections(struct objtool_file *file) if (!sec) return -1; - symids = elf_add_data(elf, sec, NULL, nr * sizeof(struct klp_symid)); + symids = elf_add_data(elf, sec, NULL, nr * sizeof(struct klp_symid), true); if (!symids) return -1; -- 2.54.0
